feat(export): async, multi-service export (#1042)

* feat(export): new infrastructure for multi-service export

* feat(export): set up multi-service export capabilities

At this time the behavior should be the same, but
there is an additional layer acting like a state machine
to schedule the work, determine if it's finished, and
notify the user if it is.

* chore: fix some test mocks

* chore: init queue handler

remove dev fxa secret env vars
(they don't exist)

* chore: add dynamodb permissions

* fix: re-throw unhandled error for sqs poller

* chore(temp): add dev data

* fix(export): use attribute substitution for reserved keywords

in DynamoDB

feat(terraform): Add new method for subscribing SQS to multiple SNS

Need to be able to update the SQS access policy to allow two ARNs;
creating subscriptions with existing method results with the policy
being chosen randomly since only one can be applied to a queue.

Since we are already using the wrapped helper stacks, just make
a new one. Alternatively just write it all manually.

* feat(export): export for annotations and shareable-lists

pulling out shared code into package

* feat(export): consolidate on using shared package

* fix: set AWS_REGION variable

* chore: initialize background tasks

* fix(export): shareable-lists queue env var name

* fix(export): shareable-list source in event bridge rule

* fix(export): add s3 bucket perms for shareable-lists

* fix: fix-mismatches

* chore: remove dev data

* fix(export): remove illegal char from dynamo attribute

* fix: re-request dynamo item with consistency

* fix: isComplete array iterator

* chore: lint

* fix(test): fix test command and wrong env var

Author: kschelonka

.docker/aws-resources/account-data-deleter.sh

@@ -3,7 +3,9 @@ set -x
 
 SQS=(
 pocket-account-data-delete-queue
+pocket-export-request-queue
 pocket-list-export-queue
+pocket-annotations-export-queue
 pocket-list-import-batch-queue
 pocket-list-import-file-queue
 )

.docker/aws-resources/shareable-lists-api.sh

@@ -1,3 +1,11 @@
+SQS=(
+pocket-shareablelist-export-queue
+)
+
+for sqs_queue in "${SQS[@]}"; do
+  awslocal sqs create-queue --queue-name "${sqs_queue}"
+done
+
 #!/bin/bash
 set -x
 bash "$(dirname "${BASH_SOURCE[0]}")/eventbus.sh"

.docker/mysql-8-resources/schema/01d-mono-db-readitla_ril-tmp-schema-list_autoincrement_id.sql

@@ -0,0 +1,19 @@
+CREATE DATABASE IF NOT EXISTS `readitla_ril-tmp`;
+
+USE `readitla_ril-tmp`;
+
+ALTER TABLE `list`
+ADD CONSTRAINT `old_primary` UNIQUE (`user_id`,`item_id`)
+;
+
+ALTER TABLE `list`
+DROP PRIMARY KEY
+;
+
+--  The same schema as list, except uses expanded types for item_id and resolved_id.
+ALTER TABLE `list` 
+  MODIFY `item_id` bigint unsigned NOT NULL,
+  MODIFY `resolved_id` bigint unsigned NOT NULL,
+  ADD COLUMN `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+  ADD PRIMARY KEY (`id`);
+

.github/workflows/aws-utils.yml

@@ -0,0 +1,28 @@
+name: AWS Utilities
+on:
+  pull_request:
+    paths:
+      - 'packages/aws-utils/**'
+      - 'docker-compose.yml'
+      - 'pnpm-lock.yaml'
+      - '.github/actions/**'
+      - '.github/workflows/aws-utils.yml'
+      - '.github/workflows/reuse-test-integrations.yml'
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - 'packages/aws-utils/**'
+      - 'docker-compose.yml'
+      - 'pnpm-lock.yaml'
+      - '.github/actions/**'
+      - '.github/workflows/aws-utils.yml'
+      - '.github/workflows/reuse-test-integrations.yml'
+jobs:
+  test-integrations:
+    if: github.event_name == 'pull_request'
+    uses: ./.github/workflows/reuse-test-integrations.yml
+    with:
+      scope: '@pocket-tools/aws-utils'
+    secrets: inherit

infrastructure/account-data-deleter/src/config/index.ts

@@ -37,6 +37,8 @@ export const config = {
     databasePort: '3306',
     sqsBatchDeleteQueueName: `${prefix}-Sqs-Batch-Delete-Consumer-Queue`,
     listExportQueueName: `${prefix}-List-Export`,
+    exportRequestQueueName: `${prefix}-Export-Request`,
+    annotationsExportQueueName: `${prefix}-Annotations-Export`,
     listImportFileQueue: `${prefix}-List-Import-Files`,
     listImportBatchQueue: `${prefix}-List-Import-Batches`,
     databaseTz: 'US/Central',
@@ -48,6 +50,7 @@ export const config = {
     snsTopicName: {
       userEvents: `PocketEventBridge-${environment}-UserEvents`,
       listEvents: `PocketEventBridge-${environment}-ListEvents`,
+      exportUpdateEvents: `PocketEventBridge-${environment}-ListExportReadyEvents`,
     },
     batchDeleteLambda: {
       name: 'BatchDeleteLambda',

infrastructure/account-data-deleter/src/dataDeleterApp.ts

@@ -7,6 +7,7 @@ import {
   dataAwsRegion,
   sqsQueue,
   dataAwsSnsTopic,
+  dynamodbTable,
 } from '@cdktf/provider-aws';
 import { S3Bucket } from '@cdktf/provider-aws/lib/s3-bucket';
 
@@ -20,10 +21,13 @@ export type DataDeleterAppConfig = {
   secretsManagerKmsAlias: dataAwsKmsAlias.DataAwsKmsAlias;
   snsTopic: dataAwsSnsTopic.DataAwsSnsTopic;
   batchDeleteQueue: sqsQueue.SqsQueue;
+  exportRequestQueue: sqsQueue.SqsQueue;
+  annotationsExportQueue: sqsQueue.SqsQueue;
   listExportQueue: sqsQueue.SqsQueue;
   listExportBucket: S3Bucket;
   listExportPartsPrefix: string;
   listExportArchivesPrefix: string;
+  exportStateDb: dynamodbTable.DynamodbTable;
   importFileQueue: sqsQueue.SqsQueue;
   importBatchQueue: sqsQueue.SqsQueue;
   listImportBucket: S3Bucket;
@@ -63,6 +67,24 @@ export class DataDeleterApp extends Construct {
       `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:${config.prefix}/*`,
     ];
 
+    // Don't pull these secrets unless in production (they don't exist in dev)
+    const FxaEnvVars = config.isProd
+      ? [
+          {
+            name: 'FXA_CLIENT_ID',
+            valueFrom: `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Web/${config.environment}/FIREFOX_WEB_AUTH_CLIENT_ID`,
+          },
+          {
+            name: 'FXA_CLIENT_SECRET',
+            valueFrom: `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Web/${config.environment}/FIREFOX_WEB_AUTH_CLIENT_SECRET`,
+          },
+          {
+            name: 'FXA_OAUTH_URL',
+            valueFrom: `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Web/${config.environment}/FIREFOX_AUTH_OAUTH_URL`,
+          },
+        ]
+      : [];
+
     const app = new PocketALBApplication(this, 'application', {
       alarms: {
         http5xxErrorPercentage: {
@@ -114,10 +136,22 @@ export class DataDeleterApp extends Construct {
               name: 'SQS_BATCH_DELETE_QUEUE_URL',
               value: `https://sqs.${region.name}.amazonaws.com/${caller.accountId}/${config.envVars.sqsBatchDeleteQueueName}`,
             },
+            {
+              name: 'EXPORT_REQUEST_QUEUE_URL',
+              value: `https://sqs.${region.name}.amazonaws.com/${caller.accountId}/${config.envVars.exportRequestQueueName}`,
+            },
+            {
+              name: 'EXPORT_REQUEST_STATE_TABLE',
+              value: this.config.exportStateDb.name,
+            },
             {
               name: 'SQS_LIST_EXPORT_QUEUE_URL',
               value: `https://sqs.${region.name}.amazonaws.com/${caller.accountId}/${config.envVars.listExportQueueName}`,
             },
+            {
+              name: 'SQS_ANNOTATIONS_EXPORT_QUEUE_URL',
+              value: `https://sqs.${region.name}.amazonaws.com/${caller.accountId}/${config.envVars.annotationsExportQueueName}`,
+            },
             {
               name: 'SQS_IMPORT_BATCH_QUEUE_URL',
               value: this.config.importBatchQueue.url,
@@ -217,18 +251,7 @@ export class DataDeleterApp extends Construct {
               name: 'EXPORT_SIGNEDURL_USER_SECRET_KEY',
               valueFrom: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:${config.name}/${config.environment}/EXPORT_USER_CREDS:secretAccessKey::`,
             },
-            {
-              name: 'FXA_CLIENT_ID',
-              valueFrom: `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Web/${config.environment}/FIREFOX_WEB_AUTH_CLIENT_ID`,
-            },
-            {
-              name: 'FXA_CLIENT_SECRET',
-              valueFrom: `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Web/${config.environment}/FIREFOX_WEB_AUTH_CLIENT_SECRET`,
-            },
-            {
-              name: 'FXA_OAUTH_URL',
-              valueFrom: `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Web/${config.environment}/FIREFOX_AUTH_OAUTH_URL`,
-            },
+            ...FxaEnvVars,
           ],
         },
       ],
@@ -276,8 +299,10 @@ export class DataDeleterApp extends Construct {
             ],
             resources: [
               this.config.batchDeleteQueue.arn,
-              this.config.listExportQueue.arn,
               this.config.importFileQueue.arn,
+              this.config.listExportQueue.arn,
+              this.config.exportRequestQueue.arn,
+              this.config.annotationsExportQueue.arn,
             ],
             effect: 'Allow',
           },
@@ -312,6 +337,19 @@ export class DataDeleterApp extends Construct {
             ],
             effect: 'Allow',
           },
+          // DynamoDB Status
+          {
+            actions: [
+              'dynamodb:DescribeTable',
+              'dynamodb:Get*',
+              'dynamodb:UpdateItem',
+            ],
+            resources: [
+              this.config.exportStateDb.arn,
+              `${this.config.exportStateDb.arn}/*`,
+            ],
+            effect: 'Allow',
+          },
         ],
         taskExecutionDefaultAttachmentArn:
           'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy',