Added Google Cloud Storage (GCS) integration for image uploads (#533)

Ref https://linear.app/ghost/issue/AP-1062/

Summary of changes:-

- Integrated Google Cloud Storage (GCS) for handling image uploads.
- Added a new Docker Compose service fake-gcs to simulate GCS locally.
- Configured environment variables in local/test services to connect with the emulator.
- Added @google-cloud/storage as a new dependency.
- Validation on boot:
-- Initializes the GCS client
-- Validates and creates the bucket (if running with the emulator)
- Introduced a new POST API: ./ghost/activitypub/upload/image
-- Validates file type and size
-- Uploads the image to the GCS bucket with a unique path
-- Returns a public URL to access the image
- Added unit tests for the upload logic.
- Added new Cucumber step definitions and a test scenario to verify the upload flow end-to-end.

Author: vershwal

.gitignore

@@ -75,3 +75,6 @@ typings/
 dist/
 
 cedar/data-generation/output/*
+
+# Fake GCS data
+fake-gcs/storage*

docker-compose.yml

@@ -20,6 +20,8 @@ services:
       - MQ_PUBSUB_SUBSCRIPTION_NAME=fedify-subscription
       - MQ_PUBSUB_GHOST_TOPIC_NAME=ghost-topic
       - MQ_PUBSUB_GHOST_SUBSCRIPTION_NAME=ghost-subscription
+      - GCP_BUCKET_NAME=activitypub
+      - GCP_STORAGE_EMULATOR_HOST=http://fake-gcs:4443
       - ACTIVITYPUB_COLLECTION_PAGE_SIZE=20
     command: yarn build:watch
     depends_on:
@@ -29,6 +31,8 @@ services:
         condition: service_healthy
       pubsub:
         condition: service_healthy
+      fake-gcs:
+        condition: service_healthy
 
   jaeger:
     image: jaegertracing/all-in-one:1.62.0
@@ -99,8 +103,6 @@ services:
       retries: 120
       start_period: 5s
 
-  # Testing
-
   activitypub-testing:
     networks:
       - test_network
@@ -126,13 +128,17 @@ services:
       - MQ_PUBSUB_SUBSCRIPTION_NAME=fedify-subscription
       - MQ_PUBSUB_GHOST_TOPIC_NAME=ghost-topic
       - MQ_PUBSUB_GHOST_SUBSCRIPTION_NAME=ghost-subscription
+      - GCP_BUCKET_NAME=activitypub
+      - GCP_STORAGE_EMULATOR_HOST=http://fake-gcs:4443
       - ACTIVITYPUB_COLLECTION_PAGE_SIZE=2
     command: yarn build:watch
     depends_on:
       mysql-testing:
         condition: service_healthy
       pubsub-testing:
         condition: service_healthy
+      fake-gcs:
+        condition: service_healthy
     healthcheck:
       test: "if [ ! -f /tmp/health.txt ]; then (wget --spider http://0.0.0.0:8083/ping || exit 1) && touch /tmp/health.txt ; else echo \"healthcheck already executed\"; fi"
       interval: 1s
@@ -165,6 +171,8 @@ services:
       - MYSQL_PORT=3306
       - MYSQL_DATABASE=activitypub
       - NODE_ENV=testing
+      - GCP_BUCKET_NAME=activitypub
+      - GCP_STORAGE_EMULATOR_HOST=http://fake-gcs:4443
       - TAGS
     command: /opt/activitypub/node_modules/.bin/cucumber-js
     depends_on:
@@ -174,6 +182,8 @@ services:
         condition: service_started
       activitypub-testing:
         condition: service_healthy
+      fake-gcs:
+        condition: service_healthy
 
   mysql-testing:
     networks:
@@ -242,6 +252,29 @@ services:
       - "8084:8080"
     entrypoint: [ "/docker-entrypoint.sh", "--global-response-templating", "--disable-gzip", "--verbose" ]
 
+  fake-gcs:
+    build: fake-gcs
+    container_name: fake-gcs
+    ports:
+      - "4443:4443"
+    environment:
+      - GCP_BUCKET_NAME=activitypub
+      - GCP_PROJECT_ID=activitypub
+    volumes:
+      - ./fake-gcs/storage:/storage
+    networks:
+      default:
+        aliases:
+          - fake-gcs
+      test_network:
+        aliases:
+          - fake-gcs
+    healthcheck:
+      test: "curl -f http://localhost:4443/storage/v1/b/${GCP_BUCKET_NAME}"
+      interval: 1s
+      retries: 120
+      start_period: 5s
+
 networks:
   test_network:
     driver: bridge

fake-gcs/Dockerfile

@@ -0,0 +1,11 @@
+FROM fsouza/fake-gcs-server
+
+# Install curl
+RUN apk add --no-cache curl
+
+# Copy the initialization script
+COPY start.sh /start.sh
+RUN chmod +x /start.sh
+
+# Set the entrypoint
+ENTRYPOINT ["/start.sh"]

fake-gcs/start.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# This script initializes the fake-gcs server and ensures the required bucket exists.
+# It starts the server, waits for it to be ready, and creates the bucket if it doesn't exist.
+
+# Ensure storage directory exists and has proper permissions
+mkdir -p /storage
+chmod 777 /storage
+
+# Start the fake-gcs server in the background
+fake-gcs-server -scheme http -port 4443 -external-url http://fake-gcs:4443 -data /storage &
+
+# Wait for the server to be ready
+sleep 1
+
+# Check if bucket exists and create if it doesn't
+if ! curl -s "http://localhost:4443/storage/v1/b/${GCP_BUCKET_NAME}" | grep -q "\"name\": \"${GCP_BUCKET_NAME}\""; then
+    curl -X POST \
+        -H "Content-Type: application/json" \
+        -d "{\"name\": \"${GCP_BUCKET_NAME}\"}" \
+        "http://localhost:4443/storage/v1/b?project=${GCP_PROJECT_ID}"
+fi
+
+# Keep the container running
+tail -f /dev/null

features/step_definitions/stepdefs.js

@@ -699,6 +699,26 @@ When(
     },
 );
 
+When(
+    /^an authenticated (\"(post|put)\"\s)?request is made to "(.*)" with a file$/,
+    async function (method, path) {
+        // Create a test file
+        const file = new File(['test image content'], 'test.jpg', {
+            type: 'image/jpeg',
+        });
+        const formData = new FormData();
+        formData.append('file', file);
+
+        this.response = await fetchActivityPub(
+            `http://fake-ghost-activitypub.test${path}`,
+            {
+                method: method || 'post',
+                body: formData,
+            },
+        );
+    },
+);
+
 When('an unauthenticated request is made to {string}', async function (path) {
     this.response = await fetchActivityPub(
         `http://fake-ghost-activitypub.test${path}`,
@@ -1992,6 +2012,19 @@ Then('the response contains {string} account details', async function (name) {
     assert.equal(typeof responseJson.followsMe, 'boolean');
 });
 
+Then('the response contains a file URL', async function () {
+    const responseJson = await this.response.clone().json();
+    assert(responseJson.fileUrl, 'Response should contain a fileUrl');
+    assert(
+        typeof responseJson.fileUrl === 'string',
+        'fileUrl should be a string',
+    );
+    assert(
+        responseJson.fileUrl.startsWith('http'),
+        'fileUrl should be a valid URL',
+    );
+});
+
 Then('the response contains the account details:', async function (data) {
     const responseJson = await this.response.clone().json();
 

features/upload-image.feature

@@ -0,0 +1,8 @@
+  Feature: Image Upload API
+  As an authenticated user
+  I want to upload images
+
+  Scenario: Upload an image
+    When an authenticated "post" request is made to "/.ghost/activitypub/upload/image" with a file
+    Then the request is accepted with a 200
+    And the response contains a file URL

package.json

@@ -60,6 +60,7 @@
     "@google-cloud/opentelemetry-cloud-trace-exporter": "2.4.1",
     "@google-cloud/opentelemetry-cloud-trace-propagator": "0.20.0",
     "@google-cloud/pubsub": "4.11.0",
+    "@google-cloud/storage": "7.16.0",
     "@hono/node-server": "1.14.1",
     "@js-temporal/polyfill": "0.5.1",
     "@logtape/logtape": "0.8.0",

src/app.ts

@@ -110,6 +110,7 @@ import {
     createGetThreadHandler,
     createPostPublishedWebhookHandler,
     createSearchHandler,
+    createStorageHandler,
     createUpdateAccountHandler,
     handleCreateNote,
 } from './http/api';
@@ -126,6 +127,7 @@ import {
 } from './mq/gcloud-pubsub-push/mq';
 import { PostService } from './post/post.service';
 import { type Site, SiteService } from './site/site.service';
+import { GCPStorageService } from './storage/gcloud-storage/gcp-storage.service';
 
 const logging = getLogger(['activitypub']);
 
@@ -188,6 +190,19 @@ export type ContextData = {
 
 const fedifyKv = await KnexKvStore.create(client, 'key_value');
 
+const gcpStorageService = new GCPStorageService();
+
+try {
+    logging.info('Initialising GCP storage service');
+    await gcpStorageService.init();
+    logging.info('GCP storage service initialised');
+} catch (err) {
+    logging.error('Failed to initialise GCP storage service {error}', {
+        error: err,
+    });
+    process.exit(1);
+}
+
 let queue: GCloudPubSubPushMessageQueue | undefined;
 
 if (process.env.USE_MQ === 'true') {
@@ -1054,6 +1069,11 @@ app.get(
         createGetNotificationsHandler(accountService, notificationService),
     ),
 );
+app.post(
+    '/.ghost/activitypub/upload/image',
+    requireRole(GhostRole.Owner, GhostRole.Administrator),
+    spanWrapper(createStorageHandler(accountService, gcpStorageService)),
+);
 /** Federation wire up */
 
 app.get(

src/http/api/index.ts

@@ -4,5 +4,6 @@ export * from './notification';
 export * from './note';
 export * from './post';
 export * from './search';
+export * from './storage';
 export * from './thread';
 export * from './webhook';

src/http/api/storage.ts

@@ -0,0 +1,50 @@
+import type { AccountService } from 'account/account.service';
+import { exhaustiveCheck, getError, getValue, isError } from 'core/result';
+import type { Context } from 'hono';
+import type { GCPStorageService } from 'storage/gcloud-storage/gcp-storage.service';
+
+export function createStorageHandler(
+    accountService: AccountService,
+    storageService: GCPStorageService,
+) {
+    /**
+     * Handle an upload to GCloud Storage bucket
+     */
+    return async function handleUpload(ctx: Context) {
+        const logger = ctx.get('logger');
+        const formData = await ctx.req.formData();
+        const file = formData.get('file');
+
+        if (!file || !(file instanceof File)) {
+            return new Response('No valid file provided', { status: 400 });
+        }
+
+        const account = await accountService.getAccountForSite(ctx.get('site'));
+        const result = await storageService.saveFile(file, account.uuid);
+
+        if (isError(result)) {
+            const error = getError(result);
+            switch (error) {
+                case 'file-too-large':
+                    logger.error(`File is too large: ${file.size} bytes`);
+                    return new Response('File is too large', { status: 413 });
+                case 'file-type-not-supported':
+                    logger.error(`File type ${file.type} is not supported`);
+                    return new Response(
+                        `File type ${file.type} is not supported`,
+                        { status: 415 },
+                    );
+                default:
+                    exhaustiveCheck(error);
+            }
+        }
+
+        const fileUrl = getValue(result);
+        return new Response(JSON.stringify({ fileUrl }), {
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            status: 200,
+        });
+    };
+}