Merge pull request #255 from Sofie-Automation/feat/s3

feat: Add support for S3

Author: anteeek

apps/single-app/app/expectedPackages.json

@@ -531,6 +531,8 @@ export namespace Expectation {
 			| ExpectedJSONData
 			| ExpectedFTPFile
 			| ExpectedUnspecified
+			| ExpectedS3Resource
+
 		export type Any =
 			| FileOnDisk
 			| MediaFileThumbnail
@@ -542,6 +544,8 @@ export namespace Expectation {
 			| JSONData
 			| FTPFile
 			| Unspecified
+			| S3Resource
+
 		export interface Base {
 			type: Type
 		}
@@ -559,6 +563,7 @@ export namespace Expectation {
 			JSON_DATA = 'json_data',
 			FTP_FILE = 'ftp_file',
 			UNSPECIFIED = 'unspecified',
+			S3_RESOURCE = 's3_resource',
 		}
 		type ExpectedType<T extends Base> = Partial<T> & Pick<T, 'type'>
 
@@ -674,5 +679,13 @@ export namespace Expectation {
 			type: Type.UNSPECIFIED
 		}
 		export type ExpectedUnspecified = ExpectedType<Unspecified>
+
+		export interface S3Resource extends Base {
+			type: Type.S3_RESOURCE
+			fileSize: number
+			modifiedDate: number
+		}
+
+		export type ExpectedS3Resource = ExpectedType<S3Resource>
 	}
 }

shared/packages/api/src/expectationApi.ts

@@ -410,6 +410,7 @@ export namespace Accessor {
 		| AtemMediaStore
 		| FTP
 		| KairosClip
+		| S3
 
 	export enum AccessType {
 		LOCAL_FOLDER = 'local_folder',
@@ -421,6 +422,7 @@ export namespace Accessor {
 		ATEM_MEDIA_STORE = 'atem_media_store',
 		FTP = 'ftp',
 		KAIROS_CLIP = 'kairos_clip',
+		S3 = 's3',
 	}
 
 	/** Generic (used in extends) */
@@ -574,6 +576,33 @@ export namespace Accessor {
 		/** Name/Id of the network the share exists on. Used to differ between different local networks. Leave empty if globally accessible. */
 		networkId?: string
 	}
+
+	/** Definition of access to a generic FTP/SFTP endpoint. (Read-access only) */
+	export interface S3 extends Base {
+		type: AccessType.S3
+
+		/** Identifier of the S3 bucket  */
+		bucketId: string
+
+		/** AWS Access key */
+		accessKey: string
+
+		/** AWS Secret access key */
+		secretAccessKey: string
+
+		/** AWS region of the bucket */
+		region: string
+
+		/** Base URL for the S3 bucket */
+		s3PublicBaseUrl: string
+
+		/** S3 endpoint (obligatory for non-AWS S3 deployments). If undefined, AWS S3 is assumed */
+		endpoint?: string
+
+		/** If true, forces path-style URLs (required for some S3-compatible storage solutions that use path-style URLs for buckets)
+		 * i.e. use this is bucket URL is `http://localhost/test` instead of `http://test.localhost` */
+		forcePathStyle?: boolean
+	}
 }
 /**
  * AccessorOnPackage contains interfaces for Accessor definitions that are put ON the Package.
@@ -591,6 +620,7 @@ export namespace AccessorOnPackage {
 		| AtemMediaStore
 		| FTP
 		| KairosClip
+		| S3
 
 	export interface LocalFolder extends Partial<Accessor.LocalFolder> {
 		/** Path to the file (starting from .folderPath). If not set, the filePath of the ExpectedPackage will be used */
@@ -623,9 +653,15 @@ export namespace AccessorOnPackage {
 		/** path to resource (combined with .basePath gives the full path), for example: /folder/myFile */
 		path?: string
 	}
+
 	export interface KairosClip extends Partial<Accessor.KairosClip> {
 		ref?: MediaRamRecRef | MediaStillRef
 	}
+
+	export interface S3 extends Partial<Accessor.S3> {
+		/** key of resource */
+		filePath?: string
+	}
 }
 
 export interface PackageContainerOnPackage extends Omit<PackageContainer, 'accessors'> {

shared/packages/api/src/inputApi.ts

@@ -29,6 +29,8 @@
 		"rimraf": "^5.0.5"
 	},
 	"dependencies": {
+		"@aws-sdk/client-s3": "^3.943.0",
+		"@aws-sdk/lib-storage": "^3.943.0",
 		"@parcel/watcher": "^2.3.0",
 		"@sofie-package-manager/api": "1.53.0-alpha.4",
 		"abort-controller": "^3.0.0",

shared/packages/worker/package.json

@@ -10,6 +10,7 @@ import { QuantelAccessorHandle } from './quantel'
 import { ATEMAccessorHandle } from './atem'
 import { FTPAccessorHandle } from './ftp'
 import { KairosClipAccessorHandle } from './kairosClip'
+import { S3AccessorHandle } from './s3'
 
 export function getAccessorHandle<Metadata>(
 	worker: BaseWorker,
@@ -56,6 +57,8 @@ export function getAccessorStaticHandle(accessor: AccessorOnPackage.Any) {
 		return FTPAccessorHandle
 	} else if (accessor.type === Accessor.AccessType.KAIROS_CLIP) {
 		return KairosClipAccessorHandle
+	} else if (accessor.type === Accessor.AccessType.S3) {
+		return S3AccessorHandle
 	} else {
 		assertNever(accessor.type) // Assert  so as to not forget to add an if-clause above
 		throw new Error(`Unsupported Accessor type "${accessor.type}"`)
@@ -108,6 +111,12 @@ export function isKairosClipAccessorHandle<Metadata>(
 	return accessorHandler.type === KairosClipAccessorHandle.type
 }
 
+export function isS3AccessorHandle<Metadata>(
+	accessorHandler: GenericAccessorHandle<Metadata>
+): accessorHandler is S3AccessorHandle<Metadata> {
+	return accessorHandler.type === S3AccessorHandle.type
+}
+
 /** Returns a generic value for how costly it is to use an Accessor type. A higher value means that it is more expensive to access the accessor */
 export function getAccessorCost(accessorType: Accessor.AccessType | undefined): number {
 	switch (accessorType) {
@@ -127,6 +136,7 @@ export function getAccessorCost(accessorType: Accessor.AccessType | undefined):
 			return 2
 		case Accessor.AccessType.HTTP_PROXY:
 		case Accessor.AccessType.HTTP:
+		case Accessor.AccessType.S3:
 			return 3
 
 		case undefined:

shared/packages/worker/src/worker/accessorHandlers/accessor.ts

@@ -103,6 +103,7 @@ export class GenericFileOperationsHandler {
 				for (const file of files) {
 					const filePath = path.join(dirPath, file.name)
 					const fullPath = path.join(folderPath, filePath)
+
 					if (file.isDirectory) {
 						await cleanUpDirectory(filePath, true)
 					} else if (file.lastModified && file.lastModified > 0) {