Rework identification API to be asynchronous

User issues a POST request with payload to /identify, unique request_id is generated
and returned to user. User then can issue GET request to /identify/<request_id> to get response.
This allows to scan big accounts with a lot of resources and do not exceed 30 seconds response timeout
for lambda integration with gateway.
New DDB was added for tracking status of user requests. API lambda(entrypoint) now triggers existing describe lambdas
to do a scan. Every describe lambda retrieves request_id from payload and updates corresponding
record in requests table after it finishes the scan of the region. This allows to track the progress of scan,
by comparing current progress and total number of scans (features * regions) and determine when the scan is finished.
Remediation is not supported for now.
Add "scan account <account id>" command to slack bot. Currently it supports only full account scan.

Author: ochuprykov

deployment/cf-templates/api.json

@@ -209,7 +209,7 @@
                 "HttpMethod": "ANY",
                 "Integration": {
                     "Type": "AWS_PROXY",
-                    "IntegrationHttpMethod": "ANY",
+                    "IntegrationHttpMethod": "POST",
                     "Uri": {"Fn::Join": ["",
                         ["arn:aws:apigateway:", {"Ref": "AWS::Region"}, ":lambda:path/2015-03-31/functions/", {"Fn::GetAtt": ["LambdaApi", "Arn"]}, "/invocations"]
                     ]}

deployment/cf-templates/ddb.json

@@ -428,6 +428,28 @@
                 },
                 "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "rds-unencrypted" ] ]}
             }
+        },
+        "DynamoDBApiRequests": {
+            "Type": "AWS::DynamoDB::Table",
+            "Properties": {
+                "AttributeDefinitions": [
+                    {
+                        "AttributeName": "request_id",
+                        "AttributeType": "S"
+                    }
+                ],
+                "KeySchema": [
+                    {
+                        "AttributeName": "request_id",
+                        "KeyType": "HASH"
+                    }
+                ],
+                "ProvisionedThroughput": {
+                    "ReadCapacityUnits": "10",
+                    "WriteCapacityUnits": "2"
+                },
+                "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "credentials" ] ]}
+            }
         }
     }
 }

deployment/configs/config.json

@@ -40,20 +40,25 @@
             "654321210987": "slave2"
         }
     },
+    "api": {
+        "ddb.table_name": "hammer-api-requests"
+    },
     "whitelisting_procedure_url": "",
     "credentials": {
         "ddb.table_name": "hammer-credentials"
     },
     "s3_bucket_acl": {
         "enabled": true,
         "ddb.table_name": "hammer-s3-public-bucket-acl",
+        "topic_name": "hammer-describe-s3-acl-lambda",
         "reporting": false,
         "remediation": false,
         "remediation_retention_period": 0
     },
     "secgrp_unrestricted_access": {
         "enabled": true,
         "ddb.table_name": "hammer-security-groups-unrestricted",
+        "topic_name": "hammer-describe-security-groups-lambda",
         "restricted_ports": [
             21,
             22,
@@ -75,6 +80,7 @@
     "user_inactivekeys": {
         "enabled": true,
         "ddb.table_name": "hammer-iam-user-keys-inactive",
+        "topic_name": "hammer-describe-iam-user-inactive-keys-lambda",
         "ignore_accounts": ["654321210987"],
         "inactive_criteria_days": 1,
         "reporting": false,
@@ -84,6 +90,7 @@
     "user_keysrotation": {
         "enabled": true,
         "ddb.table_name": "hammer-iam-user-keys-rotation",
+        "topic_name": "hammer-describe-iam-user-keys-rotation-lambda",
         "rotation_criteria_days": 10,
         "reporting": false,
         "remediation": false,
@@ -92,52 +99,60 @@
     "s3_bucket_policy": {
         "enabled": true,
         "ddb.table_name": "hammer-s3-public-bucket-policy",
+        "topic_name": "hammer-describe-s3-policy-lambda",
         "reporting": false,
         "remediation": false,
         "remediation_retention_period": 7
     },
     "cloudtrails": {
         "enabled": true,
         "ddb.table_name": "hammer-cloudtrails",
+        "topic_name": "hammer-describe-cloudtrails-lambda",
         "reporting": false
     },
     "ebs_unencrypted_volume": {
         "enabled": true,
         "ddb.table_name": "hammer-ebs-volumes-unencrypted",
+        "topic_name": "hammer-describe-ebs-unencrypted-volumes-lambda",
         "accounts": ["123456789012", "210987654321"],
         "reporting": false
     },
     "ebs_public_snapshot": {
         "enabled": true,
         "ddb.table_name": "hammer-ebs-snapshots-public",
+        "topic_name": "hammer-describe-ebs-public-snapshots-lambda",
         "reporting": false,
         "remediation": false,
         "remediation_retention_period": 0
     },
     "rds_public_snapshot": {
         "enabled": true,
         "ddb.table_name": "hammer-rds-public-snapshots",
+        "topic_name": "hammer-describe-rds-public-snapshots-lambda",
         "reporting": false,
         "remediation": false,
         "remediation_retention_period": 0
     },
     "sqs_public_access": {
         "enabled": true,
         "ddb.table_name": "hammer-sqs-public-access",
+        "topic_name": "hammer-describe-sqs-public-policy-lambda",
         "reporting": true,
         "remediation": false,
         "remediation_retention_period": 0
     },
     "s3_encryption": {
         "enabled": true,
         "ddb.table_name": "hammer-s3-unencrypted",
+        "topic_name": "hammer-describe-s3-encryption-lambda",
         "reporting": true,
         "remediation": false,
         "remediation_retention_period": 0
     },
     "rds_encryption": {
         "enabled": true,
         "ddb.table_name": "hammer-rds-unencrypted",
+        "topic_name": "hammer-describe-rds-encryption-lambda",
         "reporting": true
     }
 }

hammer/identification/lambdas/api/authorizer.py

@@ -28,6 +28,11 @@ def lambda_handler(event, context):
     policy.restApiId = apiGatewayArnTmp[0]
     policy.region = tmp[3]
     policy.stage = apiGatewayArnTmp[1]
+    # a quick hack to allow GET calls to /identify/{request_id}, request_id is hex string
+    # rewrite this solution to more generic variant
+    if len(apiGatewayArnTmp) == 5:
+        full_path = '/identify/' + apiGatewayArnTmp[4]
+        policy.allowMethod(HttpVerb.GET, full_path)
     policy.allowMethod(HttpVerb.POST, '/identify')
     policy.allowMethod(HttpVerb.POST, '/remediate')
 
@@ -190,4 +195,4 @@ def build(self):
         policy['policyDocument']['Statement'].extend(self._getStatementForEffect('Allow', self.allowMethods))
         policy['policyDocument']['Statement'].extend(self._getStatementForEffect('Deny', self.denyMethods))
 
-        return policy
+        return policy