Merge branch 'qat' into mod/dev-15451-submission-zip-prefix

Author: dpb-bah

dataactbroker/app.py

@@ -106,7 +106,7 @@ def before_request():
         content_type = request.headers.get("Content-Type")
 
         # If the request is a POST we want to log the request body
-        if request.method == "POST" and content_type and "login" not in request.url.lower():
+        if request.method == "POST" and content_type and "login" not in request.base_url.lower():
             request_body = {}
 
             # If request is json, turn it into a dict
@@ -137,7 +137,7 @@ def handle_exception(exception):
         return JsonResponse.error(wrapped, wrapped.status)
 
     # Add routes for modules here
-    add_login_routes(flask_app, bcrypt)
+    add_login_routes(flask_app, local, bcrypt)
 
     add_file_routes(flask_app, local, broker_file_path)
     add_generation_routes(flask_app, local, broker_file_path)

dataactbroker/handlers/account_handler.py

@@ -1,3 +1,4 @@
+import hmac
 import json
 import logging
 from operator import attrgetter
@@ -119,7 +120,7 @@ def proxy_login(self, session):
             roles = safe_dictionary.get_value("roles")
             token = safe_dictionary.get_value("token")
 
-            if token != CONFIG_BROKER["api_proxy_token"]:
+            if not hmac.compare_digest(token, CONFIG_BROKER["api_proxy_token"]):
                 raise ValueError("Invalid token")
 
             user = sess.query(User).filter(func.lower(User.email) == func.lower(email)).one_or_none()

dataactbroker/handlers/fileHandler.py

@@ -1,12 +1,13 @@
 import boto3
 import calendar
+import csv
 import logging
 import os
+import re
 import requests
-import threading
-import csv
-import tempfile
 import shutil
+import tempfile
+import threading
 
 from collections import namedtuple
 from datetime import datetime, timedelta
@@ -27,7 +28,7 @@
 )
 from dataactbroker.helpers.fabs_derivations_helper import fabs_derivations, log_derivation
 from dataactbroker.helpers.filters_helper import permissions_filter, agency_filter
-from dataactbroker.helpers.generic_helper import zip_dir
+from dataactbroker.helpers.generic_helper import sanitize_for_csv, zip_dir
 from dataactbroker.permissions import active_user_can_on_submission
 
 from dataactcore.aws.s3Handler import S3Handler
@@ -1604,7 +1605,9 @@ def update_submission_comments(submission, comment_request, is_local):
     json = comment_request or {}
     # clean input
     comments_json = {
-        key.upper(): value.strip() for key, value in json.items() if isinstance(value, str) and value.strip()
+        key.upper(): sanitize_for_csv(value.strip())
+        for key, value in json.items()
+        if isinstance(value, str) and value.strip()
     }
 
     sess = GlobalDB.db().session
@@ -2088,7 +2091,7 @@ def add_list_submission_filters(query, filters, submission_updated_view):
             # Make a list of all the names we're filtering on
             file_array = []
             for file_name in file_list:
-                file_regex = r".+\/.*" + str(file_name).upper() + r"[^\/]*$"
+                file_regex = r".+\/.*" + re.escape(str(file_name).upper()) + r"[^\/]*$"
                 file_array.append(func.upper(Job.filename).op("~")(file_regex))
 
             # Create a subquery to get all submission IDs related to upload jobs (every type except cross-file has an

dataactbroker/handlers/generation_handler.py

@@ -266,6 +266,6 @@ def check_detached_generation(job_id):
     Returns:
         Response object with keys job_id, status, file_type, url, message, start, and end.
     """
-    response_dict = generation_helper.check_file_generation(job_id)
+    response_dict = generation_helper.check_file_generation(job_id, detached=True)
 
     return JsonResponse.create(StatusCode.OK, response_dict)

dataactbroker/helpers/generation_helper.py

@@ -194,11 +194,12 @@ def start_dabs_generation(job, start_date, end_date, agency_code):
     logger.debug(log_data)
 
 
-def check_file_generation(job_id):
+def check_file_generation(job_id, detached=False):
     """Check the status of a file generation
 
     Args:
         job_id: upload Job ID
+        detached: whether the job_id provided should be detached or not
     Return:
         Dict with keys: job_id, status, file_type, message, url, start, end
     """
@@ -223,6 +224,12 @@ def check_file_generation(job_id):
         response_dict["status"] = "invalid"
         response_dict["message"] = "No generation job found with the specified ID"
         return response_dict
+    if detached and upload_job.submission_id is not None:
+        response_dict["start"] = ""
+        response_dict["end"] = ""
+        response_dict["status"] = "invalid"
+        response_dict["message"] = "Job ID not associated with a detached file generation."
+        return response_dict
 
     response_dict["file_type"] = lookups.FILE_TYPE_DICT_LETTER[upload_job.file_type_id]
     response_dict["size"] = upload_job.file_size

dataactbroker/helpers/generic_helper.py

@@ -185,6 +185,28 @@ def generate_raw_quoted_query(queryset):
     )
 
 
+def sanitize_for_csv(value):
+    """Checks for and neutralizes potential CSV formula injection, exception of numbers
+
+    Args:
+        value: string to sanitize
+
+    Returns:
+        sanitized string (single quote in front if applicable)
+    """
+    if not value or not isinstance(value, str):
+        return value
+
+    # Characters that spreadsheet software parses as the start of a formula
+    formula_chars = r"^\s*[=+\-@|](?![\d.,]+$)"
+
+    # Prepend an apostrophe if a formula-triggering character is detected
+    if re.match(formula_chars, value):
+        return f"'{value}"
+
+    return value
+
+
 def fy(raw_date):
     """Get fiscal year from date, datetime, or date string
 

dataactbroker/helpers/validation_helper.py

@@ -180,6 +180,24 @@ def clean_numbers_vectorized(series: pd.Series, convert_to_str=False):
     series.update(replacements[cleanable])
 
 
+def sanitize_csv_vectorized(series: pd.Series):
+    """In-place sanitization of strings that start with +/-/=/@, exception of numbers
+
+    Args:
+        series: the series that will be cleaned (updated in-place)
+
+    Returns:
+        None (in-place update of given Series)
+    """
+    formula_pattern = r"^\s*[=+\-@|](?![\d.,]+$)"
+
+    # Get subset of values in series that need to be sanitized
+    needs_escape = series.str.contains(formula_pattern, regex=True, na=False)
+
+    # Update in place
+    series.loc[needs_escape] = "'" + series.loc[needs_escape].astype(str)
+
+
 def concat_flex(row):
     """Concatenates the headers and contents of all the flex cells in one row of a submission and joins the list
     on commas

dataactbroker/permissions.py

@@ -224,12 +224,13 @@ def wrapped(*args, **kwargs):
     return inner
 
 
-def requires_agency_code_perms(perm):
+def requires_agency_code_perms(perm, location=None):
     """Decorator that checks the current user's permissions and validates them against the agency code. It expects an
      agency_code parameter on top of the function arguments.
 
     Args:
         perm: the type of permission we are checking for
+        location: the location to check for the agency code ('query' or 'json')
 
     Returns:
         The args/kwargs that were initially provided
@@ -243,13 +244,10 @@ def inner(fn):
         @requires_login
         @wraps(fn)
         def wrapped(*args, **kwargs):
-            for location in ["query", "json"]:
-                req_args = webargs_parser.parse(
-                    {"agency_code": webargs_fields.String(load_default=None)}, request, location=location
-                )
-                agency_code = req_args.get("agency_code", None)
-                if agency_code is not None:
-                    break
+            req_args = webargs_parser.parse(
+                {"agency_code": webargs_fields.String(load_default=None)}, request, location=location
+            )
+            agency_code = req_args.get("agency_code", None)
             # Ensure there is an agency_code
             if agency_code is None:
                 raise ResponseError("Missing required parameter: agency_code", StatusCode.CLIENT_ERROR)

dataactbroker/routes/dashboard_routes.py

@@ -54,7 +54,7 @@ def historic_dabs_graphs(**kwargs):
     @use_kwargs(
         {
             "page": webargs_fields.Int(load_default=1),
-            "limit": webargs_fields.Int(load_default=5),
+            "limit": webargs_fields.Int(validate=webargs_validate.Range(min=1, max=1000), load_default=5),
             "sort": webargs_fields.String(load_default="period"),
             "order": webargs_fields.String(load_default="desc"),
             "filters": webargs_fields.Dict(keys=webargs_fields.String(), required=True),
@@ -166,7 +166,7 @@ def significance_counts(submission, file, **kwargs):
                 load_default="warning",
             ),
             "page": webargs_fields.Int(load_default=1),
-            "limit": webargs_fields.Int(load_default=5),
+            "limit": webargs_fields.Int(validate=webargs_validate.Range(min=1, max=1000), load_default=5),
             "sort": webargs_fields.String(load_default="significance"),
             "order": webargs_fields.String(load_default="asc"),
         },

dataactbroker/routes/file_routes.py

@@ -104,7 +104,7 @@ def get_banner_list(login):
     @parser.use_kwargs(
         {
             "page": webargs_fields.Int(load_default=1),
-            "limit": webargs_fields.Int(load_default=5),
+            "limit": webargs_fields.Int(validate=webargs_validate.Range(min=1, max=1000), load_default=5),
             "published": webargs_fields.String(
                 required=True, validate=webargs_validate.OneOf(("mixed", "true", "false"))
             ),