Jules/upgrades (#7)

Author: Juliya Smith

CHANGELOG.md

@@ -8,6 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 The intended audience of this file is for py42 consumers -- as such, changes that don't affect
 how a consumer would use the library (e.g. adding unit tests, updating documentation, etc) are not captured here.
 
+## Unreleased
+
+### Added
+
+- Begin and end date now support specifying time: `code42 securitydata print -b 2020-02-02 12:00:00`.
+
 ## 0.2.0 - 2020-02-25
 
 ### Removed

setup.py

@@ -20,7 +20,7 @@
     packages=find_packages("src"),
     package_dir={"": "src"},
     python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4",
-    install_requires=["c42eventextractor==0.1.3", "keyring==18.0.1","py42==0.4.4"],
+    install_requires=["c42eventextractor==0.2.0", "keyring==18.0.1","py42==0.5.1"],
     license="MIT",
     include_package_data=True,
     zip_safe=False,

src/code42cli/date_helper.py

@@ -0,0 +1,72 @@
+from datetime import datetime, timedelta
+from c42eventextractor.common import convert_datetime_to_timestamp
+from py42.sdk.file_event_query.event_query import EventTimestamp
+
+_DEFAULT_LOOK_BACK_DAYS = 60
+_MAX_LOOK_BACK_DAYS = 90
+_FORMAT_VALUE_ERROR_MESSAGE = u"input must be a date in YYYY-MM-DD or YYYY-MM-DD HH:MM:SS format."
+
+
+def create_event_timestamp_range(begin_date=None, end_date=None):
+    min_timestamp = _parse_min_timestamp(begin_date)
+    max_timestamp = _parse_max_timestamp(end_date)
+    _verify_timestamp_order(min_timestamp, max_timestamp)
+    return EventTimestamp.in_range(min_timestamp, max_timestamp)
+
+
+def _parse_min_timestamp(begin_date_str):
+    if not begin_date_str:
+        return _get_default_min_timestamp()
+    min_timestamp = _parse_timestamp(begin_date_str)
+    boundary_date = datetime.utcnow() - timedelta(days=_MAX_LOOK_BACK_DAYS)
+    boundary = convert_datetime_to_timestamp(boundary_date)
+    if min_timestamp and min_timestamp < boundary:
+        raise ValueError(u"'Begin date' must be within 90 days.")
+    return min_timestamp
+
+
+def _parse_max_timestamp(end_date_str):
+    if not end_date_str:
+        return _get_default_max_timestamp()
+    return _parse_timestamp(end_date_str)
+
+
+def _verify_timestamp_order(min_timestamp, max_timestamp):
+    if min_timestamp is None or max_timestamp is None:
+        return
+    if min_timestamp >= max_timestamp:
+        raise ValueError(u"Begin date cannot be after end date")
+
+
+def _parse_timestamp(date_tuple):
+    try:
+        date_str = _join_date_tuple(date_tuple)
+        date_format = u"%Y-%m-%d" if len(date_tuple) == 1 else u"%Y-%m-%d %H:%M:%S"
+        time = datetime.strptime(date_str, date_format)
+    except ValueError:
+        raise ValueError(_FORMAT_VALUE_ERROR_MESSAGE)
+    return convert_datetime_to_timestamp(time)
+
+
+def _join_date_tuple(date_tuple):
+    if not date_tuple:
+        return None
+    date_str = date_tuple[0]
+    if len(date_tuple) == 1:
+        return date_str
+    if len(date_tuple) == 2:
+        date_str = "{0} {1}".format(date_str, date_tuple[1])
+    else:
+        raise ValueError(_FORMAT_VALUE_ERROR_MESSAGE)
+    return date_str
+
+
+def _get_default_min_timestamp():
+    now = datetime.utcnow()
+    start_day = timedelta(days=_DEFAULT_LOOK_BACK_DAYS)
+    days_ago = now - start_day
+    return convert_datetime_to_timestamp(days_ago)
+
+
+def _get_default_max_timestamp():
+    return convert_datetime_to_timestamp(datetime.utcnow())

src/code42cli/main.py

@@ -1,5 +1,6 @@
 from argparse import ArgumentParser
 
+from code42cli.compat import str
 from code42cli.profile import profile
 import code42cli.securitydata.main as securitydata
 

src/code42cli/profile/config.py

@@ -21,7 +21,7 @@ def get_config_profile():
         util.print_error("Profile is not set.")
         print("")
         print("To set, use: ")
-        util.print_bold("\tcode42cli profile set -s <authority-URL> -u <username>")
+        util.print_bold("\tcode42 profile set -s <authority-URL> -u <username>")
         print("")
         exit(1)
 

src/code42cli/securitydata/arguments/search.py

@@ -33,21 +33,23 @@ def _add_begin_date_arg(parser):
     parser.add_argument(
         "-b",
         "--begin",
+        nargs="+",
         action="store",
         dest=SearchArguments.BEGIN_DATE,
         help="The beginning of the date range in which to look for events, "
-        "in YYYY-MM-DD UTC format OR a number (number of minutes ago).",
+        "in YYYY-MM-DD (UTC) or YYYY-MM-DD HH:MM:SS (UTC+24-hr time) format.",
     )
 
 
 def _add_end_date_arg(parser):
     parser.add_argument(
         "-e",
         "--end",
+        nargs="+",
         action="store",
         dest=SearchArguments.END_DATE,
         help="The end of the date range in which to look for events, "
-        "in YYYY-MM-DD UTC format OR a number (number of minutes ago).",
+        "in YYYY-MM-DD (UTC) or YYYY-MM-DD HH:MM:SS (UTC+24-hr time) format.",
     )
 
 

src/code42cli/securitydata/extraction.py

@@ -1,15 +1,15 @@
 from __future__ import print_function
 import json
-from datetime import datetime, timedelta
 from py42.sdk import SDK
 from py42 import debug_level
 from py42 import settings
-from c42eventextractor.common import FileEventHandlers
-from c42eventextractor.extractors import AEDEventExtractor
-from c42eventextractor.common import convert_datetime_to_timestamp
+from c42eventextractor import FileEventHandlers
+from c42eventextractor.extractors import FileEventExtractor
 
+from code42cli.compat import str
 from code42cli.securitydata.options import ExposureType
 from code42cli.util import print_error
+from code42cli import date_helper as date_helper
 from code42cli.securitydata.cursor_store import AEDCursorStore
 from code42cli.securitydata.logger_factory import get_error_logger
 from code42cli.profile.profile import get_profile
@@ -21,7 +21,7 @@ def extract(output_logger, args):
     handlers = _create_event_handlers(output_logger, args.is_incremental)
     profile = get_profile()
     sdk = _get_sdk(profile, args.is_debug_mode)
-    extractor = AEDEventExtractor(sdk, handlers)
+    extractor = FileEventExtractor(sdk, handlers)
     _call_extract(extractor, args)
 
 
@@ -53,44 +53,13 @@ def _get_sdk(profile, is_debug_mode):
     return code42
 
 
-def _parse_min_timestamp(begin_date):
-    min_timestamp = _parse_timestamp(begin_date)
-    boundary_date = datetime.utcnow() - timedelta(days=90)
-    boundary = convert_datetime_to_timestamp(boundary_date)
-    if min_timestamp and min_timestamp < boundary:
-        print("Argument '--begin' must be within 90 days.")
-        exit(1)
-    return min_timestamp
-
-
-def _parse_timestamp(input_string):
-    try:
-        # Input represents date str like '2020-02-13'
-        time = datetime.strptime(input_string, "%Y-%m-%d")
-    except ValueError:
-        # Input represents amount of seconds ago like '86400'
-        if input_string and input_string.isdigit():
-            now = datetime.utcnow()
-            time = now - timedelta(minutes=int(input_string))
-        else:
-            raise ValueError("input must be a positive integer or a date in YYYY-MM-DD format.")
-
-    return convert_datetime_to_timestamp(time)
-
-
 def _call_extract(extractor, args):
     if not _determine_if_advanced_query(args):
-        min_timestamp = _parse_min_timestamp(args.begin_date) if args.begin_date else None
-        max_timestamp = _parse_timestamp(args.end_date) if args.end_date else None
+        event_timestamp_filter_group = _get_event_timestamp_filter(args)
         _verify_exposure_types(args.exposure_types)
-        _verify_timestamp_order(min_timestamp, max_timestamp)
-        extractor.extract(
-            initial_min_timestamp=min_timestamp,
-            max_timestamp=max_timestamp,
-            exposure_types=args.exposure_types,
-        )
+        extractor.extract(args.exposure_types, event_timestamp_filter_group)
     else:
-        extractor.extract_raw(args.advanced_query)
+        extractor.extract_advanced(args.advanced_query)
 
 
 def _determine_if_advanced_query(args):
@@ -99,7 +68,7 @@ def _determine_if_advanced_query(args):
         for key in given_args:
             val = given_args[key]
             if not _verify_compatibility_with_advanced_query(key, val):
-                print_error("You cannot use --advanced-query with additional search args.")
+                print_error(u"You cannot use --advanced-query with additional search args.")
                 exit(1)
         return True
     return False
@@ -113,20 +82,19 @@ def _verify_compatibility_with_advanced_query(key, val):
     return True
 
 
+def _get_event_timestamp_filter(args):
+    try:
+        return date_helper.create_event_timestamp_range(args.begin_date, args.end_date)
+    except ValueError as ex:
+        print_error(str(ex))
+        exit(1)
+
+
 def _verify_exposure_types(exposure_types):
     if exposure_types is None:
         return
     options = list(ExposureType())
     for exposure_type in exposure_types:
         if exposure_type not in options:
-            print_error("'{0}' is not a valid exposure type.".format(exposure_type))
+            print_error(u"'{0}' is not a valid exposure type.".format(exposure_type))
             exit(1)
-
-
-def _verify_timestamp_order(begin_timestamp, end_timestamp):
-    if begin_timestamp is None or end_timestamp is None:
-        return
-
-    if begin_timestamp >= end_timestamp:
-        print_error("Begin date cannot be after end date")
-        exit(1)

src/code42cli/securitydata/logger_factory.py

@@ -2,14 +2,15 @@
 import logging
 from logging.handlers import RotatingFileHandler
 from c42eventextractor.logging.formatters import (
-    AEDDictToJSONFormatter,
-    AEDDictToCEFFormatter,
-    AEDDictToRawJSONFormatter,
+    FileEventDictToJSONFormatter,
+    FileEventDictToCEFFormatter,
+    FileEventDictToRawJSONFormatter,
 )
-from c42eventextractor.logging.handlers import NoPrioritySysLogHandler
+from c42eventextractor.logging.handlers import NoPrioritySysLogHandlerWrapper
 
+from code42cli.compat import str
 from code42cli.securitydata.options import OutputFormat
-from code42cli.util import get_user_project_path
+from code42cli.util import get_user_project_path, print_error
 
 
 def get_logger_for_stdout(output_format):
@@ -35,7 +36,7 @@ def get_logger_for_server(hostname, protocol, output_format):
     if _logger_has_handlers(logger):
         return logger
 
-    handler = NoPrioritySysLogHandler(hostname, protocol=protocol)
+    handler = NoPrioritySysLogHandlerWrapper(hostname, protocol=protocol).handler
     return _init_logger(logger, handler, output_format)
 
 
@@ -62,15 +63,19 @@ def _init_logger(logger, handler, output_format):
 
 
 def _apply_logger_dependencies(logger, handler, formatter):
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
+    try:
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    except Exception as ex:
+        print_error(str(ex))
+        exit(1)
     return logger
 
 
 def _get_formatter(output_format):
     if output_format == OutputFormat.JSON:
-        return AEDDictToJSONFormatter()
+        return FileEventDictToJSONFormatter()
     elif output_format == OutputFormat.CEF:
-        return AEDDictToCEFFormatter()
+        return FileEventDictToCEFFormatter()
     else:
-        return AEDDictToRawJSONFormatter()
+        return FileEventDictToRawJSONFormatter()

src/code42cli/securitydata/subcommands/clear_checkpoint.py

@@ -12,7 +12,7 @@ def init(subcommand_parser):
 
 def clear_checkpoint(*args):
     """Removes the stored checkpoint that keeps track of the last event you got.
-        To use, do `code42cli clear-checkpoint`.
+        To use, run `code42 clear-checkpoint`.
         This affects `incremental` mode by causing it to behave like it has never been run before.
     """
     AEDCursorStore().reset()

tests/conftest.py

@@ -1,4 +1,6 @@
 import pytest
+import json as json_module
+from datetime import datetime, timedelta
 from argparse import Namespace
 
 from code42cli.profile.config import ConfigurationKeys
@@ -47,3 +49,34 @@ class ConfigParserMocks(object):
     section_adder = None
     reader = None
     sections = None
+
+
+def get_first_filter_value_from_json(json):
+    return json_module.loads(str(json))["filters"][0]["value"]
+
+
+def get_second_filter_value_from_json(json):
+    return json_module.loads(str(json))["filters"][1]["value"]
+
+
+def parse_date_from_first_filter_value(json):
+    date_str = get_first_filter_value_from_json(json)
+    return convert_str_to_date(date_str)
+
+
+def parse_date_from_second_filter_value(json):
+    date_str = get_second_filter_value_from_json(json)
+    return convert_str_to_date(date_str)
+
+
+def convert_str_to_date(date_str):
+    return datetime.strptime(date_str, u"%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+def get_test_date(days_ago):
+    now = datetime.utcnow()
+    return now - timedelta(days=days_ago)
+
+
+def get_test_date_str(days_ago):
+    return get_test_date(days_ago).strftime("%Y-%m-%d")