Merge pull request #2376 from mitre/fix_flake8_test

Resolve flake8 errors

Author: wbooth

app/api/packs/advanced.py

@@ -22,8 +22,6 @@ async def enable(self):
         self.app_svc.application.router.add_route('GET', '/advanced/configurations', self._section_configurations)
         self.app_svc.application.router.add_route('GET', '/advanced/exfills', self._section_exfil_files)
 
-    """ PRIVATE """
-
     @check_authorization
     @template('planners.html')
     async def _section_planners(self, request):

app/api/packs/campaign.py

@@ -21,8 +21,6 @@ async def enable(self):
         self.app_svc.application.router.add_route('GET', '/campaign/profiles', self._section_profiles)
         self.app_svc.application.router.add_route('GET', '/campaign/operations', self._section_operations)
 
-    """ PRIVATE """
-
     @check_authorization
     @template('agents.html')
     async def _section_agent(self, request):
@@ -110,8 +108,6 @@ def load_usage_markdown(header):
         return dict(operations=operations, groups=groups, adversaries=adversaries, sources=sources, planners=planners,
                     obfuscators=obfuscators, usage=usage)
 
-    """ PRIVATE """
-
     @staticmethod
     def _rollup_abilities(abilities):
         rolled = defaultdict(list)

app/api/rest_api.py

@@ -43,8 +43,6 @@ async def enable(self):
         self.app_svc.application.router.add_route('GET', '/api/{index}', self.rest_core_info)
         self.app_svc.application.router.add_route('GET', '/file/download_exfil', self.download_exfil_file)
 
-    """ BOILERPLATE """
-
     @template('login.html', status=401)
     async def login(self, request):
         return dict()
@@ -65,8 +63,6 @@ async def landing(self, request):
         data = dict(plugins=[p.display for p in plugins], errors=self.app_svc.errors + self._request_errors(request))
         return render_template('%s.html' % access[0].name, request, data)
 
-    """ API ENDPOINTS """
-
     @check_authorization
     async def rest_core(self, request):
         try:
@@ -165,8 +161,6 @@ def is_in_exfil_dir(f):
                 return web.HTTPNotFound(body=str(e))
         return web.HTTPBadRequest(body='A file needs to be specified for download')
 
-    """ PRIVATE """
-
     @staticmethod
     def _request_errors(request):
         errors = []

app/api/v2/handlers/base_object_api.py

@@ -23,8 +23,6 @@ def __init__(self, description, obj_class, schema, ram_key, id_property, auth_sv
     def add_routes(self, app: web.Application):
         pass
 
-    """GET"""
-
     async def get_all_objects(self, request: web.Request):
         access = await self.get_request_permissions(request)
 
@@ -48,8 +46,6 @@ async def get_object(self, request: web.Request):
 
         return self._api_manager.dump_object_with_filters(obj, include, exclude)
 
-    """POST"""
-
     async def create_object(self, request: web.Request):
         data = await request.json()
 
@@ -75,8 +71,6 @@ async def _error_if_object_with_id_exists(self, obj_id: str):
             if self._api_manager.find_object(self.ram_key, search):
                 raise JsonHttpBadRequest(f'{self.description.capitalize()} with given id already exists: {obj_id}')
 
-    """PATCH"""
-
     async def update_object(self, request: web.Request):
         data, access, obj_id, query, search = await self._parse_common_data_from_request(request)
 
@@ -94,8 +88,6 @@ async def update_on_disk_object(self, request: web.Request):
             raise JsonHttpNotFound(f'{self.description.capitalize()} not found: {obj_id}')
         return obj
 
-    """PUT"""
-
     async def create_or_update_object(self, request: web.Request):
         data, access, obj_id, query, search = await self._parse_common_data_from_request(request)
 
@@ -120,8 +112,6 @@ async def create_or_update_on_disk_object(self, request: web.Request):
 
         return obj
 
-    """DELETE"""
-
     async def delete_object(self, request: web.Request):
         obj_id = request.match_info.get(self.id_property)
 
@@ -141,8 +131,6 @@ async def delete_on_disk_object(self, request: web.Request):
         obj_id = request.match_info.get(self.id_property)
         await self._api_manager.remove_object_from_disk_by_id(identifier=obj_id, ram_key=self.ram_key)
 
-    """Helpers"""
-
     async def _parse_common_data_from_request(self, request) -> (dict, dict, str, dict, dict):
         data = {}
         raw_body = await request.read()

app/api/v2/handlers/operation_api.py

@@ -163,8 +163,6 @@ async def get_potential_links_by_paw(self, request: web.Request):
         potential_links = await self._api_manager.get_potential_links(operation_id, access, paw)
         return web.json_response(potential_links)
 
-    '''Overridden Methods'''
-
     async def create_object(self, request: web.Request):
         data = await request.json()
         await self._error_if_object_with_id_exists(data.get(self.id_property))
@@ -178,8 +176,6 @@ async def update_object(self, request: web.Request):
             raise JsonHttpNotFound(f'{self.description.capitalize()} not found: {obj_id}')
         return obj
 
-    ''' PRIVATE '''
-
     async def _read_output_parameter_(self, request: web.Request):
         raw_body = await request.read()
         output = False

app/api/v2/managers/ability_api_manager.py

@@ -45,7 +45,6 @@ async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_pro
         await self._save_and_reload_object(file_path, existing_obj_data, obj_class, obj.access)
         return next(self.find_objects(ram_key, {id_property: obj_id}))
 
-    '''Helpers'''
     def _validate_ability_data(self, create: bool, data: dict):
         # Correct ability_id key for ability file saving.
         data['id'] = data.pop('ability_id', '')

app/api/v2/managers/base_api_manager.py

@@ -23,8 +23,6 @@ def __init__(self, data_svc, file_svc, logger=None):
     def log(self):
         return self._log
 
-    """Object Retrieval"""
-
     def find_objects(self, ram_key: str, search: dict = None):
         """Find objects matching the given criteria"""
         for obj in self._data_svc.ram[ram_key]:
@@ -54,8 +52,6 @@ def dump_object_with_filters(obj: Any, include: List[str] = None, exclude: List[
                 dumped.pop(exclude_attribute, None)
         return dumped
 
-    """Object Creation"""
-
     def create_object_from_schema(self, schema: SchemaMeta, data: dict, access: BaseWorld.Access):
         obj_schema = schema()
         obj = obj_schema.load(data)
@@ -79,8 +75,6 @@ def _get_allowed_from_access(self, access) -> BaseWorld.Access:
         else:
             return self._data_svc.Access.RED
 
-    """Object Updates"""
-
     def find_and_update_object(self, ram_key: str, data: dict, search: dict = None):
         for obj in self.find_objects(ram_key, search):
             new_obj = self.update_object(obj, data)
@@ -112,17 +106,13 @@ async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_pro
         await self._save_and_reload_object(file_path, existing_obj_data, obj_class, obj.access)
         return next(self.find_objects(ram_key, {id_property: obj_id}))
 
-    """"Object Replacement"""
-
     async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str):
         obj_id = getattr(obj, id_property)
         file_path = await self._get_existing_object_file_path(obj_id, ram_key)
 
         await self._save_and_reload_object(file_path, data, type(obj), obj.access)
         return next(self.find_objects(ram_key, {id_property: obj_id}))
 
-    """"Object Removal"""
-
     async def remove_object_from_memory_by_id(self, identifier: str, ram_key: str, id_property: str):
         await self._data_svc.remove(ram_key, {id_property: identifier})
 
@@ -132,8 +122,6 @@ async def remove_object_from_disk_by_id(self, identifier: str, ram_key: str):
         if os.path.exists(file_path):
             os.remove(file_path)
 
-    """Helpers"""
-
     @staticmethod
     async def _get_new_object_file_path(identifier: str, ram_key: str) -> str:
         """Create file path for new object"""

app/api/v2/managers/operation_api_manager.py

@@ -122,7 +122,6 @@ async def get_potential_links(self, operation_id: str, access: dict, paw: str =
         potential_links = [potential_link.display for potential_link in operation.potential_links]
         return potential_links
 
-    """Object Creation Helpers"""
     async def get_operation_object(self, operation_id: str, access: dict):
         try:
             operation = (await self._data_svc.locate('operations', {'id': operation_id}))[0]

app/contacts/contact_gist.py

@@ -141,8 +141,6 @@ async def get_uploads(self):
             self.log.error('Receiving file uploads over c2 (%s) failed: %s' % (self.__class__.__name__, e))
             return []
 
-    """ PRIVATE """
-
     async def _send_instructions(self, agent, instructions):
         response = dict(paw=agent.paw,
                         sleep=await agent.calculate_sleep(),

app/contacts/contact_html.py

@@ -16,8 +16,6 @@ def __init__(self, services):
     async def start(self):
         self.app_svc.application.router.add_route('*', self.get_config('app.contact.html'), self._accept_beacon)
 
-    """ PRIVATE """
-
     @template('weather.html')
     async def _accept_beacon(self, request):
         try:

app/contacts/contact_http.py

@@ -17,8 +17,6 @@ def __init__(self, services):
     async def start(self):
         self.app_svc.application.router.add_route('POST', '/beacon', self._beacon)
 
-    """ PRIVATE """
-
     async def _beacon(self, request):
         try:
             profile = json.loads(self.contact_svc.decode_bytes(await request.read()))

app/contacts/contact_slack.py

@@ -150,8 +150,6 @@ async def get_uploads(self):
             self.log.error('Receiving file uploads over c2 (%s) failed: %s' % (self.__class__.__name__, e))
             return []
 
-    """ PRIVATE """
-
     async def _send_instructions(self, agent, instructions):
         response = dict(paw=agent.paw,
                         sleep=await agent.calculate_sleep(),

app/contacts/contact_tcp.py

@@ -93,8 +93,6 @@ async def send(self, session_id: int, cmd: str, timeout: int = 60) -> Tuple[int,
             self.log.exception(e)
             return 1, '~$ ', str(e), ''
 
-    """ PRIVATE """
-
     @staticmethod
     async def _handshake(reader):
         profile_bites = (await reader.readline()).strip()

app/objects/c_agent.py

@@ -332,8 +332,6 @@ def assign_pending_executor_change(self):
         self._executor_change_to_assign = None
         return executor_change
 
-    """ PRIVATE """
-
     def _replace_payload_data(self, decoded_cmd, file_svc):
         for uuid in re.findall(self.RESERVED['payload'], decoded_cmd):
             if self.is_uuid4(uuid):

app/objects/c_operation.py

@@ -337,8 +337,6 @@ async def write_event_logs_to_disk(self, file_svc, data_svc, output=False):
         await self._write_logs_to_disk(event_logs, file_name, event_logs_dir, file_svc)
         logging.debug('Wrote event logs for operation %s to disk at %s/%s' % (self.name, event_logs_dir, file_name))
 
-    """ PRIVATE """
-
     async def _write_logs_to_disk(self, logs, file_name, dest_dir, file_svc):
         logs_dumps = json.dumps(logs)
         await file_svc.save_file(file_name, logs_dumps.encode(), dest_dir, encrypt=False)

app/objects/c_planner.py

@@ -62,8 +62,6 @@ async def which_plugin(self):
                 return plugin
         return None
 
-    """ PRIVATE """
-
     @staticmethod
     def _set_stopping_conditions(conditions):
         if conditions:

app/objects/c_plugin.py

@@ -84,8 +84,6 @@ async def expand(self, services):
         except Exception as e:
             logging.error('Error expanding plugin=%s, %s' % (self.name, e))
 
-    """ PRIVATE """
-
     def _load_module(self):
         try:
             return import_module('plugins.%s.hook' % self.name)

app/objects/secondclass/c_link.py

@@ -223,8 +223,6 @@ def replace_origin_link_id(self):
         decoded_cmd = self.decode_bytes(self.command)
         self.command = self.encode_string(decoded_cmd.replace(self.RESERVED['origin_link_id'], self.id))
 
-    """ PRIVATE """
-
     def _emit_status_change_event(self, from_status, to_status):
         event_svc = BaseService.get_service('event_svc')
 

app/service/app_svc.py

@@ -193,8 +193,6 @@ def register_subapp(self, path: str,  app: web.Application):
     def get_loaded_plugins(self):
         return tuple(self._loaded_plugins)
 
-    """ PRIVATE """
-
     async def _save_configurations(self, main_config_file='default'):
         for cfg_name, cfg_file in [('main', main_config_file), ('agents', 'agents'), ('payloads', 'payloads')]:
             with open('conf/%s.yml' % cfg_file, 'w') as config:

app/service/contact_svc.py

@@ -93,8 +93,6 @@ async def get_tunnel(self, name):
         tunnel = [t for t in self.tunnels if t.name == name]
         return tunnel[0] if len(tunnel) > 0 else None
 
-    """ PRIVATE """
-
     async def _sanitize_paw(self, input_paw):
         """
         Remove any characters from the given paw that do not fall in the following set:

app/service/data_svc.py

@@ -241,8 +241,6 @@ async def load_yaml_file(self, object_class, filename, access):
             obj.access = access
             await self.store(obj)
 
-    """ PRIVATE """
-
     async def _load(self, plugins=()):
         try:
             async_tasks = []

app/service/file_svc.py

@@ -201,8 +201,6 @@ def add_xored_extension(filename):
             return filename
         return '%s.xored' % filename
 
-    """ PRIVATE """
-
     def _save(self, filename, content, encrypt=True):
         if encrypt and (self.encryptor and self.encrypt_output):
             content = bytes(FILE_ENCRYPTION_FLAG, 'utf-8') + self.encryptor.encrypt(content)

app/service/learning_svc.py

@@ -50,8 +50,6 @@ async def learn(self, facts, link, blob, operation=None):
         await update_scores(operation=None, increment=len(found_facts), used=facts, facts=link.facts)
         await self._store_results(link, found_facts, operation)
 
-    """ PRIVATE """
-
     @staticmethod
     async def _save_fact(link, facts, fact, operation=None):
         fact.origin_type = OriginType.LEARNED

app/service/planning_svc.py

@@ -274,8 +274,6 @@ async def sort_links(links):
         """
         return sorted(links, key=lambda k: (-k.score))
 
-    """ PRIVATE """
-
     async def _stop_bucket_exhaustion(self, planner, operation, condition_stop):
         """Determine whether to continue running the bucket.
 

app/service/rest_svc.py

@@ -341,8 +341,6 @@ async def build_potential_links(self, operation, agents, abilities):
                 potential_links.append(pl)
         return await self.get_service('planning_svc').sort_links(potential_links)
 
-    """ PRIVATE """
-
     async def _build_operation_object(self, access, data):
         name = data.pop('name')
         group = data.pop('group', '')

app/utility/base_planning_svc.py

@@ -166,8 +166,6 @@ async def obfuscate_commands(self, agent, obfuscator, links):
             s_link.command = self.encode_string(mod.run(s_link))
         return links
 
-    """ PRIVATE """
-
     @staticmethod
     def _list_historic_duplicate_singletons(operation):
         """