Local PoC sync via Copybara - 58599c0b45cecf8b2f0547a1afd87544800b8519 Add verbose flag to twine upload command in CodeArtifact ... by Iman Shafiei <[email protected]>

  - 521521fcef242de9a2579e2871023ce1fcd9469a Fix CodeArtifact login domain and domain owner in workflow by Iman Shafiei <[email protected]>
  - 63da05a7c83b9e1e4acfd24bdfe88f59bd7c9e0c Refactor CodeArtifact workflow to install twine and publi... by Iman Shafiei <[email protected]>
  - 835cf35efbee67948948287552851d90985f45b5 Fix CodeArtifact repository configuration by swapping tok... by Iman Shafiei <[email protected]>
  - b910eddd8f9279339ba94aac29b2f3341c90325e Add logging for CodeArtifact URL and token length in work... by Iman Shafiei <[email protected]>
  - 59fd410dd5cb319a1d759ecce42331d21a12081e Add retrieval of authorization token for CodeArtifact in ... by Iman Shafiei <[email protected]>
  - 79245720d5edae81b0f7f174a017526521fd1276 Add step to configure Poetry repository with CodeArtifact... by Iman Shafiei <[email protected]>
  - bb0d752a2c9997827b2cd0674f413022bb4f6e45 Refactor CodeArtifact workflow to use Python slim image a... by Iman Shafiei <[email protected]>
  - 138066b9170b8a5c26cb75f1f639cf635836143e Add bulk insert and bulk search services to the Earth Dai... by Iman Shafiei <[email protected]>
  - b922f64bc514c34a0cde19981c54a2981ded6d1c More debugging by Iman Shafiei <[email protected]>
  - 63d60e2dd8db8ed4033bc21dce1fa0ced37af67d Update container image for CodeArtifact upload job to use... by Iman Shafiei <[email protected]>
  - c9cf80878ef51c245e02dd55517d7a1941dca989 Add debug step for AWS environment in CodeArtifact workflow by Iman Shafiei <[email protected]>
  - 275a4ec9d6f6dc2ede041fa6f0afa6591426f3b6 Fix formatting of docstring in generate_token.py and upda... by Iman Shafiei <[email protected]>
  - 083edc6eba06d8c5c9bbae5e720b6dfb1c2269d3 Update permissions and clean up workflow steps for CodeAr... by Iman Shafiei <[email protected]>
  - ad1ce501dadf4aa0bfb48c14c06aaea9f10a07fe Add GitHub Actions workflow for publishing to AWS CodeArt... by Iman Shafiei <[email protected]>
  - 22f380c1b7cd594a93d0a355a39d9c92d80ffa27 Refactor internal service initialization and add error ha... by Iman Shafiei <[email protected]>
  - 6d6a64deb92793a5ecc52a7bb6bc305cc47d80c3 Refactor job runner selection for private and public repo... by Iman Shafiei <[email protected]>
  - 082fa1a4f29cb050ee4ea53c88c06075cad3f26a Fix runner selection logic for lint job based on reposito... by Iman Shafiei <[email protected]>
  - 5be4aaa4d335f5892ec6a0e8af923f5b4e875590 Add pull_request trigger to workflows for better CI integ... by Iman Shafiei <[email protected]>
  - 6522b429410df79b2c6acf02fc842dbc4f8f2427 Fix virtualenv activation command in Copybara sync workflow by Iman Shafiei <[email protected]>
  - 1822e48ca9b79a2f57a73d16e4554e5c87f99d89 Refactor secret scanning workflow to clone repository man... by Iman Shafiei <[email protected]>
  - 73a6795b16bcf2f74c0269a1577cdce695e00747 Refactor secret scanning workflow to use Alpine image and... by Iman Shafiei <[email protected]>
  - 3b0e46719f0810c291cb8e336314d52bd2847879 Add step to install jq in secret scanning workflow by Iman Shafiei <[email protected]>
  - bd142eae532db672b6e061ed3d97a55f3c886129 Add configuration for Git safe directory in secret scanni... by Iman Shafiei <[email protected]>
  - ab8515a7030c0fb9aea7eb0b34ef4a168ddfe9d9 Add configuration for Git safe directory in secret scanni... by Iman Shafiei <[email protected]>
  - 4cc3959122cc10b7d01529813e654e1a9fa1a575 Refactor secret scanning workflow to use TruffleHog Docke... by Iman Shafiei <[email protected]>
  - feaa54f1ce823560a2f8ce7c611dc99b107b35fb Update secret scanning workflow to install additional dep... by Iman Shafiei <[email protected]>
  - d770a696c4a6c94f9935ab947bc35d73caa19a3d Update secret scanning workflow to use Docker stable imag... by Iman Shafiei <[email protected]>
  - 30c5bdc77bba8a0395941f20275ad5abeb71293d Update secret scanning workflow to install additional dep... by Iman Shafiei <[email protected]>
  - e38fab2d223599461f7deb7a247d6b4ae590d228 Add configuration for Git safe directory in secret scanni... by Iman Shafiei <[email protected]>
  - c1f13aaeb118143d22d54268c9a363941323a4a9 Add git installation step to secret scanning workflow by Iman Shafiei <[email protected]>
  - c20fcc1b8adea1cc67ff7bc6691ad7a80c864250 Add container configuration with Ubuntu image to CI workf... by Iman Shafiei <[email protected]>
  - 2c0b4062c69e6bd71beca6632670b0f77d05611e format with ruff 0.11.0 by Iman Shafiei <[email protected]>
  - b5efc402a65bdac855632248c4fddf23ca16d4ca Refactor CI test jobs to use a matrix strategy for Python... by Iman Shafiei <[email protected]>
  - 3485721d1cb2b8d3eb57e9b962125e5cc1abef1a Refactor linting workflow to consolidate Python version h... by Iman Shafiei <[email protected]>
  - 88fa81508552f3658534fb96850a80b6ded31b92 Add CI workflows for linting and testing with Python vers... by Iman Shafiei <[email protected]>
  - ba318c0b5667f9585ec50045f3ea5f9254f8f0a9 Update CI workflows to use ed-runner-set for consistency ... by Iman Shafiei <[email protected]>
  - 5effcd5b81839d7f68783d2a1ce65144cfa8e5b8 Update CI workflows and BulkInsert/BulkSearch services fo... by Iman Shafiei <[email protected]>
  - ea78f2c91cb69f3b55d8125a7b17b5f157b4bfa7 Add InternalService and bulk operations support to EDSCli... by Iman Shafiei <[email protected]>
  - bfe7972f2b1df36143c609736e5ff952ff149403 Comment out CodeQL workflow configuration in codeql.yml by Iman Shafiei <[email protected]>
  - 0e46cd33ede3f3f1a8b8ad059a64069cf1ab6aa4 Beta/add pyjwt dependency and implement GitHub token gene... by Iman Shafiei <[email protected]>
  - a9174ac2cf3c1455019f98efba2dfb355980ede8 Merge remote-tracking branch 'public-repo/beta/1.x' by Iman Shafiei <[email protected]>
  - dbdaa9c820ec7245f5964ead6ce985803157e92a Initial commit by edawader <[email protected]>

GitOrigin-RevId: 58599c0b45cecf8b2f0547a1afd87544800b8519

Author: Iman

.github/workflows/ci.yaml

@@ -6,14 +6,14 @@ on:
       - main
       - beta/*
   pull_request:
-    branches:
-      - main
-      - beta/*
+  workflow_dispatch:
   schedule:
     - cron: '31 17 * * 5'
 
 jobs:
   analyze:
+    if: ${{ !github.event.repository.private }}  # Only run on public repos
+
     name: Analyze (${{ matrix.language }})
     runs-on: 'ubuntu-latest'
     permissions:

.github/workflows/codeql.yml

@@ -0,0 +1,38 @@
+name: Lint, Format and Type Check
+
+on:
+  push:
+    branches:
+      - main
+      - beta/*
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  lint:
+    runs-on: ${{ github.repository_visibility == 'private' && 'ed-runner-set' || 'ubuntu-latest' }}
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    container:
+      image: python:${{ matrix.python-version }}
+    
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install system dependencies
+        run: |
+          apt-get update
+          apt-get install -y gdal-bin libgdal-dev
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Install test dependencies
+        run: poetry install --only tests
+
+      - name: Run Linting
+        run: poetry run tox -e lint
+
+      - name: Run Formatting Check
+        run: poetry run tox -e format

.github/workflows/lint.yml

@@ -7,8 +7,12 @@ on:
 
 jobs:
   pypi-publish:
+    if: ${{ !github.event.repository.private }}  # Only run on public repos
+
     name: Upload release to PyPI
     runs-on: ubuntu-latest
+    container:
+      image: ubuntu:latest
     environment:
       name: pypi
       url: https://pypi.org/p/earthdaily
@@ -19,7 +23,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: "3.13"
 
       - name: Install Poetry
         run: pip install poetry

.github/workflows/publish_pypi_on_tag.yaml

@@ -0,0 +1,52 @@
+name: Publish to CodeArtifact
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  publish-to-codeartifact:
+    name: Upload to AWS CodeArtifact
+    runs-on: [ed-runner-set]
+    container:
+      image: python:3.13-slim
+
+    steps:
+      - name: Install system dependencies
+        run: |
+          apt-get update
+          apt-get install -y curl unzip gnupg awscli build-essential
+
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Clean old dist
+        run: rm -rf dist/
+
+      - name: Build package with Poetry
+        run: poetry build
+
+      - name: Login to CodeArtifact
+        run: |
+          aws codeartifact login \
+            --tool twine \
+            --repository pypi \
+            --domain urthecast \
+            --domain-owner ${{ secrets.FGPROD_ACCOUNT_ID }} \
+            --region us-east-1
+
+      - name: Install twine and publish to CodeArtifact
+        run: |
+          pip install twine
+          twine upload --repository codeartifact dist/* --verbose

.github/workflows/publish_to_codeartifact.yaml

@@ -1,26 +1,33 @@
-name: "Secret Scanning"
+name: Secret Scanning
 
 on:
   push:
     branches:
       - main
       - beta/*
   pull_request:
-    branches:
-      - main
-      - beta/*
   workflow_dispatch:
 
 jobs:
-  secret-scan:
-    name: Scan for Secrets
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      security-events: write
+  security-secrets:
+    name: TruffleHog Secret Scan
+    runs-on: ${{ github.repository_visibility == 'private' && 'ed-runner-set' || 'ubuntu-latest' }}
+    container:
+      image: alpine:latest
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
+      - name: Install Git and dependencies
+        run: |
+          apk add --no-cache git curl jq
+
+      - name: Clone the repository manually
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git clone https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git repo
+          cd repo
+
+          # install TruffleHog
+          curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin
 
-      - name: Run TruffleHog OSS
-        uses: trufflesecurity/trufflehog@v3.88.17
+          # run TruffleHog scan
+          trufflehog filesystem "." --results=verified,unknown --fail --json | jq

.github/workflows/secret-scanning.yml

@@ -0,0 +1,45 @@
+name: earthdaily-python-client CI
+
+on:
+  push:
+    branches:
+      - main
+      - beta/*
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: Unit Tests (Python ${{ matrix.python-version }})
+    runs-on: ${{ github.repository_visibility == 'private' && 'ed-runner-set' || 'ubuntu-latest' }}
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        include:
+          - python-version: "3.10"
+            tox_env: py310
+          - python-version: "3.11"
+            tox_env: py311
+          - python-version: "3.12"
+            tox_env: py312
+          - python-version: "3.13"
+            tox_env: py313
+    container:
+      image: python:${{ matrix.python-version }}
+    
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install GDAL and dependencies
+        run: |
+          apt-get update
+          apt-get install -y gdal-bin libgdal-dev
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Install test dependencies
+        run: poetry install --only tests
+
+      - name: Run tests with tox
+        run: poetry run tox -e ${{ matrix.tox_env }}

.github/workflows/test.yaml

@@ -4,6 +4,13 @@
 from earthdaily.legacy import EarthDataStore
 from earthdaily.platform import PlatformService
 
+try:
+    from earthdaily.internal import InternalService
+
+    _HAS_INTERNAL = True
+except ImportError:
+    _HAS_INTERNAL = False
+
 
 class EDSClient:
     """
@@ -78,3 +85,23 @@ def legacy(self):
         if not hasattr(self, "_legacy_service"):
             self._legacy_service = EarthDataStore(self.api_requester)
         return self._legacy_service
+
+    @property
+    def internal(self):
+        """
+        Lazily initializes and returns the InternalService for interacting with internal API endpoints.
+
+        Returns:
+        -------
+        InternalService:
+            The service that interacts with internal API operations.
+        Raises:
+        -------
+        NotImplementedError:
+            If the internal module is not available in this build of EDSClient.
+        """
+        if not _HAS_INTERNAL:
+            raise NotImplementedError("The 'internal' module is not available in this build of EDSClient.")
+        if not hasattr(self, "_internal_service"):
+            self._internal_service = InternalService(self.api_requester)
+        return self._internal_service

earthdaily/_eds_client.py

@@ -2,6 +2,8 @@
 from pystac_client.stac_api_io import StacApiIO
 
 from earthdaily._api_requester import APIRequester
+from earthdaily.platform._bulk_insert import BulkInsertService
+from earthdaily.platform._bulk_search import BulkSearchService
 
 
 class PlatformService:
@@ -26,6 +28,8 @@ def __init__(self, api_requester: APIRequester, pre_sign_urls: bool):
             A flag indicating whether to use pre-signed URLs for asset access.
         """
         self.api_requester = api_requester
+        self.bulk_search = BulkSearchService(api_requester)
+        self.bulk_insert = BulkInsertService(api_requester)
 
         self.pystac_client = Client.open(
             f"{api_requester.base_url}/platform/v1/stac",