From 81abffbe9401fa1c4e1bf14b85a9f1030ee333d5 Mon Sep 17 00:00:00 2001 From: f-allian Date: Mon, 5 Jan 2026 16:35:51 +0000 Subject: [PATCH 1/8] fix: updated DAFNI config and workflows --- .github/workflows/publish-to-dafni.yaml | 20 +- dafni/.env | 19 +- dafni/Dockerfile | 11 +- dafni/README.md | 47 ++--- dafni/data/inputs/causal_tests.json | 79 -------- dafni/data/outputs/causal_tests_results.json | 176 ------------------ dafni/entrypoint.sh | 52 ++++++ dafni/main_dafni.py | 0 dafni/model_definition.yaml | 118 +++++++++++- .../causal_test_results.json | 6 +- 10 files changed, 227 insertions(+), 301 deletions(-) delete mode 100644 dafni/data/inputs/causal_tests.json delete mode 100644 dafni/data/outputs/causal_tests_results.json create mode 100644 dafni/entrypoint.sh delete mode 100644 dafni/main_dafni.py diff --git a/.github/workflows/publish-to-dafni.yaml b/.github/workflows/publish-to-dafni.yaml index 051655de..b255c821 100644 --- a/.github/workflows/publish-to-dafni.yaml +++ b/.github/workflows/publish-to-dafni.yaml @@ -18,8 +18,10 @@ jobs: - name: Get release version id: get_release run: | - # Extract the version without the 'v' prefix using regex - export VERSION=$(curl -sSL https://api.github.com/repos/CITCOM-project/CausalTestingFramework/releases/latest | jq -r '.tag_name' | sed -E 's/^v?([0-9]+(\.[0-9]+)?).*/\1/') + # Extract only the major version from the latest release tag + export VERSION=$(curl -sSL https://api.github.com/repos/CITCOM-project/CausalTestingFramework/releases/latest \ + | jq -r '.tag_name' \ + | sed -E 's/^v?([0-9]+).*/\1/') echo "version=$VERSION" >> $GITHUB_OUTPUT build_and_upload: @@ -35,6 +37,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 with: python-version: '3.10' @@ -45,22 +48,19 @@ jobs: python -m pip install causal-testing-framework -e .[dev] python -m pip install dafni-cli - - name: Check if unit tests are passing first - id: tests - uses: ./.github/workflows/ci-tests.yml - - name: Build the container - if: success() && steps.tests.outcome == 'success' run: | docker build -t ctf:${{ env.VERSION }} -f ./dafni/Dockerfile . docker save ctf:${{ env.VERSION }} | gzip > ctf-dafni-${{ env.VERSION }}.tar.gz - - name: Log into DAFNI run: | dafni login - name: Upload to DAFNI run: | - dafni upload model ./dafni/model_definition.yaml ctf-dafni-${{ env.VERSION }}.tar.gz --version-message "Causal Testing Framework v${{ env.VERSION }}. Uploaded via Github Actions." --parent-id ${{ env.DAFNI_PARENT_ID }} -y - dafni logout \ No newline at end of file + dafni upload model ./dafni/model_definition.yaml \ + ctf-dafni-${{ env.VERSION }}.tar.gz \ + --version-message "Causal Testing Framework v${{ env.VERSION }}. Uploaded via Github Actions." \ + --parent-id ${{ env.DAFNI_PARENT_ID }} -y + dafni logout diff --git a/dafni/.env b/dafni/.env index 3143724b..9cd6cbeb 100644 --- a/dafni/.env +++ b/dafni/.env @@ -1,5 +1,20 @@ #.env -VARIABLES_PATH=./data/inputs/variables.json +DAG_PATH=./data/inputs/dag.dot CAUSAL_TESTS=./data/inputs/causal_tests.json DATA_PATH=./data/inputs/runtime_data.csv -DAG_PATH=./data/inputs/dag.dot \ No newline at end of file +CAUSAL_TEST_RESULTS=./data/outputs/causal_test_results.json + +EXECUTION_MODE=auto +ESTIMATOR=LinearRegressionEstimator +EFFECT_TYPE=direct +ESTIMATE_TYPE=coefficient +THREADS=0 +IGNORE_CYCLES=false +VERBOSE=false +QUERY= +ADEQUACY=false +BOOTSTRAP_SIZE=100 +SILENT=false +BATCH_SIZE=0 + + diff --git a/dafni/Dockerfile b/dafni/Dockerfile index 2330a9df..acb6a541 100644 --- a/dafni/Dockerfile +++ b/dafni/Dockerfile @@ -9,7 +9,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 # Label maintainer -LABEL maintainer="Dr. Farhad Allian - The University of Sheffield" +LABEL maintainer="Dr. Neil Walkinshaw - The University of Sheffield" # Create a folder for the source code/outputs RUN mkdir -p ./causal_testing @@ -19,11 +19,12 @@ RUN mkdir -p ./data/outputs COPY --chown=nobody ./causal_testing ./causal_testing COPY --chown=nobody ./dafni/data/inputs ./data/inputs +# Copy your entrypoint script +COPY dafni/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + # Install core dependencies using PyPi RUN pip install causal-testing-framework --no-cache-dir -# Set the PYTHONPATH environment variable to include the /src directory -ENV PYTHONPATH="/causal_testing:${PYTHONPATH}" - # Define the entrypoint/commands -CMD python -m causal_testing --dag_path $DAG_PATH --data_path $DATA_PATH --test_config $TESTS_PATH --output $OUTPUT \ No newline at end of file +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/dafni/README.md b/dafni/README.md index 47f0b186..7f24065d 100644 --- a/dafni/README.md +++ b/dafni/README.md @@ -1,33 +1,36 @@ # Causal Testing Framework on DAFNI - This directory contains the containerisation files of the causal testing framework using Docker, which is used -to upload the framework onto [DAFNI](https://www.dafni.ac.uk). + to upload the framework onto [DAFNI](https://www.dafni.ac.uk). - It is **not** recommended to install the causal testing framework using Docker, and should only be installed - using [PyPI](https://pypi.org/project/causal-testing-framework/). + using [conda-forge](https://anaconda.org/channels/conda-forge/packages/causal-testing-framework/overview) or [PyPI](https://pypi.org/project/causal-testing-framework/). ### Directory Hierarchy -- `data` contains two sub-folders (the structure is important for DAFNI). - - `inputs` is a folder that contains the input files that are (separately) uploaded to DAFNI. - - `causal_tests.json` is a JSON file that contains the causal tests. - - `dag.dot` is a dot file that contains the directed acyclic graph (dag). In this file, Causal Variables are defined as - node metadata attributes as key-value pairs using the following syntax: - `node [datatype="int", typestring="input"]`. The `datatype` key specifies the datatype of the causal variable - as a string (e.g. `"int"`, `"str"`) and the `typestring` key specifies its typestring, which is also a string - representing the variable type (e.g. `"input"` or `"output"`). - - `runtime_data.csv` is the `.csv` file that contains the runtime data. +- `data` contains two folders (structure is critical for DAFNI workflows): + - `inputs` contains all input files that are uploaded to DAFNI. + - `causal_tests.json` is a JSON file containing generated causal tests. If it exists, the framework can automatically run tests without regenerating. + - `dag.dot` is a DOT file defining the directed acyclic graph (DAG). Causal variables are stored in node metadata as key-value pairs using the syntax: + `node [datatype="int", typestring="input"]` + - `datatype` specifies the variable's data type (e.g., `"int"`, `"str"`). + - `typestring` specifies whether the variable is an `"input"` or `"output"`. + - `runtime_data.csv` contains the input runtime data for testing. + - `outputs` is the folder where `causal_test_results.json` is created after running tests. - - `outputs` is a folder where the `causal_tests_results.json` output file is created. +### Workflow -### Docker files -- `model_definition.yaml` is the model metadata that is required to be uploaded to DAFNI. -- `Dockerfile` is the main blueprint that builds the image. The main command calls the `causal_testing` module, - with specified paths for the DAG, input runtime data, test configurations, and the output filename as defined above. - This command is identical to that referenced in the main [README.md](../README.md) file. -- `docker-compose.yaml` is another method of building the image and running the container in one line. - Note: the `.env` file that contains the environment variables for `main_dafni.py` is only used here. -- `.dockerignore` tells the Dockerfile which files to not include in the image. -- `.env` is an example of a configuration file containing the environment variables. This is only required - if using `docker-compose` to build the image. +- The `entrypoint.sh` script now supports auto-detection: + - If `causal_tests.json` exists in `data/inputs`, the script automatically runs the test mode. + - If it does not exist, the script generates the causal tests first. + - The user can still override this behaviour by setting `EXECUTION_MODE` explicitly in the `.env` file. +- Filenames for `causal_tests.json` and `causal_test_results.json` are now configurable through environment variables (`CAUSAL_TESTS` and `CAUSAL_TEST_RESULTS`) in the `.env` file. +- Input/output directories are fixed as `data/inputs` and `data/outputs` to comply with DAFNI requirements. +- The script now reads all configuration parameters (estimator, effect type, threads, verbosity, query filters, adequacy metrics, etc.) **from the `.env` file**, keeping the Docker image and container clean and flexible. +### Docker files +- `model_definition.yaml` is the model metadata required for DAFNI. +- `Dockerfile` builds the container image and uses `entrypoint.sh` as the main entrypoint. All paths and options are now configurable via `.env`. +- `docker-compose.yaml` allows building and running the container with a single command. The `.env` file is required here to define all environment variables. +- `.dockerignore` specifies files to exclude from the Docker image. +- `.env` provides all configurable environment variables for the workflow (execution mode, filenames, estimator options, DAG/effects configuration, and runtime options). This is only needed if using `docker-compose`. \ No newline at end of file diff --git a/dafni/data/inputs/causal_tests.json b/dafni/data/inputs/causal_tests.json deleted file mode 100644 index 539dd927..00000000 --- a/dafni/data/inputs/causal_tests.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "tests": [ - { - "name": "vaccine --> cum_vaccinations", - "estimator": "LinearRegressionEstimator", - "estimate_type": "coefficient", - "effect": "direct", - "treatment_variable": "vaccine", - "expected_effect": { - "cum_vaccinations": "SomeEffect" - }, - "formula": "cum_vaccinations ~ vaccine", - "skip": false - }, - { - "name": "vaccine --> cum_vaccinated", - "estimator": "LinearRegressionEstimator", - "estimate_type": "coefficient", - "effect": "direct", - "treatment_variable": "vaccine", - "expected_effect": { - "cum_vaccinated": "SomeEffect" - }, - "formula": "cum_vaccinated ~ vaccine", - "skip": false - }, - { - "name": "vaccine --> cum_infections", - "estimator": "LinearRegressionEstimator", - "estimate_type": "coefficient", - "effect": "direct", - "treatment_variable": "vaccine", - "expected_effect": { - "cum_infections": "SomeEffect" - }, - "formula": "cum_infections ~ vaccine", - "skip": false - }, - { - "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']", - "estimator": "LinearRegressionEstimator", - "estimate_type": "coefficient", - "effect": "direct", - "treatment_variable": "cum_vaccinations", - "expected_effect": { - "cum_vaccinated": "NoEffect" - }, - "formula": "cum_vaccinated ~ cum_vaccinations + vaccine", - "alpha": 0.05, - "skip": false - }, - { - "name": "cum_vaccinations _||_ cum_infections | ['vaccine']", - "estimator": "LinearRegressionEstimator", - "estimate_type": "coefficient", - "effect": "direct", - "treatment_variable": "cum_vaccinations", - "expected_effect": { - "cum_infections": "NoEffect" - }, - "formula": "cum_infections ~ cum_vaccinations + vaccine", - "alpha": 0.05, - "skip": false - }, - { - "name": "cum_vaccinated _||_ cum_infections | ['vaccine']", - "estimator": "LinearRegressionEstimator", - "estimate_type": "coefficient", - "effect": "direct", - "treatment_variable": "cum_vaccinated", - "expected_effect": { - "cum_infections": "NoEffect" - }, - "formula": "cum_infections ~ cum_vaccinated + vaccine", - "alpha": 0.05, - "skip": false - } - ] -} \ No newline at end of file diff --git a/dafni/data/outputs/causal_tests_results.json b/dafni/data/outputs/causal_tests_results.json deleted file mode 100644 index 00dcbc6c..00000000 --- a/dafni/data/outputs/causal_tests_results.json +++ /dev/null @@ -1,176 +0,0 @@ -[ - { - "name": "vaccine --> cum_vaccinations", - "estimate_type": "coefficient", - "effect": "direct", - "treatment_variable": "vaccine", - "expected_effect": { - "cum_vaccinations": "SomeEffect" - }, - "formula": "cum_vaccinations ~ vaccine", - "alpha": 0.05, - "skip": false, - "passed": true, - "result": { - "treatment": "vaccine", - "outcome": "cum_vaccinations", - "adjustment_set": [], - "effect_measure": "coefficient", - "effect_estimate": { - "vaccine": 315785.1333333332 - }, - "ci_low": [ - 315339.1666500188 - ], - "ci_high": [ - 316231.1000166476 - ] - } - }, - { - "name": "vaccine --> cum_vaccinated", - "estimate_type": "coefficient", - "effect": "direct", - "treatment_variable": "vaccine", - "expected_effect": { - "cum_vaccinated": "SomeEffect" - }, - "formula": "cum_vaccinated ~ vaccine", - "alpha": 0.05, - "skip": false, - "passed": true, - "result": { - "treatment": "vaccine", - "outcome": "cum_vaccinated", - "adjustment_set": [], - "effect_measure": "coefficient", - "effect_estimate": { - "vaccine": 266389.91666666657 - }, - "ci_low": [ - 265943.93821015797 - ], - "ci_high": [ - 266835.89512317517 - ] - } - }, - { - "name": "vaccine --> cum_infections", - "estimate_type": "coefficient", - "effect": "direct", - "treatment_variable": "vaccine", - "expected_effect": { - "cum_infections": "SomeEffect" - }, - "formula": "cum_infections ~ vaccine", - "alpha": 0.05, - "skip": false, - "passed": true, - "result": { - "treatment": "vaccine", - "outcome": "cum_infections", - "adjustment_set": [], - "effect_measure": "coefficient", - "effect_estimate": { - "vaccine": 3332.883333333332 - }, - "ci_low": [ - 3274.9650508109467 - ], - "ci_high": [ - 3390.801615855717 - ] - } - }, - { - "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']", - "estimate_type": "coefficient", - "effect": "direct", - "treatment_variable": "cum_vaccinations", - "expected_effect": { - "cum_vaccinated": "NoEffect" - }, - "formula": "cum_vaccinated ~ cum_vaccinations + vaccine", - "alpha": 0.05, - "skip": false, - "passed": false, - "result": { - "treatment": "cum_vaccinations", - "outcome": "cum_vaccinated", - "adjustment_set": [ - "vaccine" - ], - "effect_measure": "coefficient", - "effect_estimate": { - "cum_vaccinations": 0.9998656401531605 - }, - "ci_low": [ - 0.9929245394499968 - ], - "ci_high": [ - 1.0068067408563242 - ] - } - }, - { - "name": "cum_vaccinations _||_ cum_infections | ['vaccine']", - "estimate_type": "coefficient", - "effect": "direct", - "treatment_variable": "cum_vaccinations", - "expected_effect": { - "cum_infections": "NoEffect" - }, - "formula": "cum_infections ~ cum_vaccinations + vaccine", - "alpha": 0.05, - "skip": false, - "passed": true, - "result": { - "treatment": "cum_vaccinations", - "outcome": "cum_infections", - "adjustment_set": [ - "vaccine" - ], - "effect_measure": "coefficient", - "effect_estimate": { - "cum_vaccinations": -0.006416682407515084 - }, - "ci_low": [ - -0.05663010083886572 - ], - "ci_high": [ - 0.043796736023835554 - ] - } - }, - { - "name": "cum_vaccinated _||_ cum_infections | ['vaccine']", - "estimate_type": "coefficient", - "effect": "direct", - "treatment_variable": "cum_vaccinated", - "expected_effect": { - "cum_infections": "NoEffect" - }, - "formula": "cum_infections ~ cum_vaccinated + vaccine", - "alpha": 0.05, - "skip": false, - "passed": true, - "result": { - "treatment": "cum_vaccinated", - "outcome": "cum_infections", - "adjustment_set": [ - "vaccine" - ], - "effect_measure": "coefficient", - "effect_estimate": { - "cum_vaccinated": -0.006176900588291234 - }, - "ci_low": [ - -0.05639349612119588 - ], - "ci_high": [ - 0.04403969494461341 - ] - } - } -] \ No newline at end of file diff --git a/dafni/entrypoint.sh b/dafni/entrypoint.sh new file mode 100644 index 00000000..9fd298e5 --- /dev/null +++ b/dafni/entrypoint.sh @@ -0,0 +1,52 @@ +#!/bin/sh +set -e + +# Auto-detect mode if EXECUTION_MODE=auto +if [ "$EXECUTION_MODE" = "auto" ]; then + if [ -f "$CAUSAL_TESTS" ]; then + EXECUTION_MODE="test" + echo "Auto mode: causal_tests.json found -> running TEST mode" + else + EXECUTION_MODE="generate" + echo "Auto mode: No causal tests found -> running GENERATE mode" + fi +else + echo "Execution mode explicitly set to: $EXECUTION_MODE" +fi + +# -------------------------- +# Generate mode +# -------------------------- +if [ "$EXECUTION_MODE" = "generate" ]; then + echo "Running causal_testing GENERATE..." + python -m causal_testing generate \ + --dag-path "$DAG_PATH" \ + --output "$CAUSAL_TESTS" \ + --estimator "$ESTIMATOR" \ + --effect-type "$EFFECT_TYPE" \ + --estimate-type "$ESTIMATE_TYPE" \ + $( [ "$IGNORE_CYCLES" = "true" ] && echo "--ignore-cycles" ) \ + --threads "$THREADS" + +# -------------------------- +# Test mode +# -------------------------- +elif [ "$EXECUTION_MODE" = "test" ]; then + if [ ! -f "$CAUSAL_TESTS" ]; then + echo "Error: Causal tests file not found at $CAUSAL_TESTS" + exit 1 + fi + + echo "Running causal_testing TEST..." + python -m causal_testing test \ + --dag-path "$DAG_PATH" \ + --data-paths "$DATA_PATH" \ + --test-config "$CAUSAL_TESTS" \ + --output "$CAUSAL_TEST_RESULTS" \ + $( [ "$IGNORE_CYCLES" = "true" ] && echo "--ignore-cycles" ) \ + $( [ "$VERBOSE" = "true" ] && echo "--verbose" ) \ + $( [ -n "$QUERY" ] && echo "--query" "$QUERY" ) \ + $( [ "$ADEQUACY" = "true" ] && echo "--adequacy --adequacy-bootstrap-size $BOOTSTRAP_SIZE" ) \ + $( [ "$SILENT" = "true" ] && echo "--silent" ) \ + $( [ "$BATCH_SIZE" != "0" ] && echo "--batch-size $BATCH_SIZE" ) +fi \ No newline at end of file diff --git a/dafni/main_dafni.py b/dafni/main_dafni.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dafni/model_definition.yaml b/dafni/model_definition.yaml index a9e2e2ce..80cf3722 100644 --- a/dafni/model_definition.yaml +++ b/dafni/model_definition.yaml @@ -16,17 +16,127 @@ metadata: the anticipated cause-effect relationships amongst the inputs and outputs of the system-under-test and the supporting mathematical framework to design statistical procedures capable of making causal inferences. Each causal test case focuses on the causal effect of an intervention made to the system-under test. - contact_point_name: Farhad Allian - contact_point_email: farhad.allian@sheffield.ac.uk + contact_point_name: Neil Walkinshaw + contact_point_email: n.walkinshaw@sheffield.ac.uk source_code: https://github.com/CITCOM-project/CausalTestingFramework licence: https://github.com/CITCOM-project/CausalTestingFramework?tab=MIT-1-ov-file#readme rights: If you use our model, please use the following instructions to cite our work - The paper citation should be the Causal Testing Framework paper, and the software citation should contain the specific Figshare DOI of the version used in your work. project_name: CITCOM + project_url: https://sites.google.com/sheffield.ac.uk/citcom/home funding: The Causal Testing Framework is supported by the UK's Engineering and Physical Sciences Research Council (EPSRC), with the project name CITCOM - "Causal Inference for Testing of Computational Models" under the grant EP/T030526/1. spec: + command: ["/bin/sh", "entrypoint.sh"] inputs: + parameters: + - name: EXECUTION_MODE + title: Execution Mode + description: > + Whether to generate causal tests from a DAG ('generate'), run + existing causal tests ('test'), or auto-detect based on causal_tests.json existence in dataslots. + type: string + required: false + default: auto + options: + - name: auto + title: Auto-detect + - name: generate + title: Generate Causal Tests + - name: test + title: Run Causal Tests + + - name: CAUSAL_TESTS + title: Causal Tests Filename + description: Filename for the causal tests JSON when running generate mode (or auto mode when generating). + type: string + default: causal_tests.json + required: false + + - name: CAUSAL_TEST_RESULTS + title: Causal Test Results Filename + description: Filename for the causal test results JSON when running test mode (or auto mode when testing). + type: string + default: causal_test_results.json + required: false + + - name: ESTIMATOR + title: Estimator + description: Name of the estimator class for evaluating causal effects. Generate mode only. + type: string + default: LinearRegressionEstimator + required: false + + - name: EFFECT_TYPE + title: Effect Type + description: Type of causal effect to estimate ('direct' or 'total'). Generate mode only. + type: string + default: direct + required: false + + - name: ESTIMATE_TYPE + title: Estimate Type + description: Type of estimate to compute when evaluating causal tests (e.g., coefficient). Generate mode only. + type: string + default: coefficient + required: false + + - name: THREADS + title: Number of Threads + description: Number of parallel threads to use for causal test generation. 0 disables parallelism. Generate mode only. + type: integer + default: 0 + required: false + + - name: IGNORE_CYCLES + title: Ignore Cycles + description: If true, cycles in the DAG are ignored. Generation mode only + type: boolean + default: false + required: false + + - name: VERBOSE + title: Verbose Logging + description: Enable verbose logging during causal test execution. Test mode only. + type: boolean + default: false + required: false + + - name: QUERY + title: Data Query + description: Optional query string to filter input data (e.g., "age > 18"). Test mode only. + type: string + default: None + required: false + + - name: ADEQUACY + title: Calculate Test Adequacy + description: If true, compute causal test adequacy metrics for each test case. Test mode only. + type: boolean + default: false + required: false + + - name: BOOTSTRAP_SIZE + title: Adequacy Bootstrap Size + description: Number of bootstrap samples for adequacy computation. Defaults to 100. Test mode only. + type: integer + default: 100 + required: false + + - name: SILENT + title: Silent Mode + description: If true, errors are recorded as results instead of causing the run to fail. Test mode only. + type: boolean + default: false + required: false + + - name: BATCH_SIZE + title: Batch Size + description: Run tests in batches of the specified size. 0 disables batching. Test mode only. + type: integer + default: 0 + required: false + dataslots: - name: Runtime csv data description: > @@ -46,11 +156,11 @@ spec: - name: Causal tests description: > - A .JSON file containing the input causal tests to be used + A .JSON file containing the input causal tests to be used. This file can also be generated using the input DAG. default: - 6f2f7c1f-81b4-4804-8f86-cca304dc7f66 path: inputs/ - required: true + required: false outputs: datasets: diff --git a/docs/source/tutorials/vaccinating_elderly/causal_test_results.json b/docs/source/tutorials/vaccinating_elderly/causal_test_results.json index e67cf0e1..88fafed0 100644 --- a/docs/source/tutorials/vaccinating_elderly/causal_test_results.json +++ b/docs/source/tutorials/vaccinating_elderly/causal_test_results.json @@ -247,13 +247,13 @@ ], "effect_measure": "coefficient", "effect_estimate": { - "cum_vaccinated": 0.0017107387725947554 + "cum_vaccinated": 0.0017107387725956436 }, "ci_low": { - "cum_vaccinated": -0.03931269141189859 + "cum_vaccinated": -0.03931269141189769 }, "ci_high": { - "cum_vaccinated": 0.0427341689570881 + "cum_vaccinated": 0.04273416895708898 } } } From 947ac76db05ebf7676cdb67ff9d584edb8f60a54 Mon Sep 17 00:00:00 2001 From: f-allian Date: Mon, 5 Jan 2026 16:37:14 +0000 Subject: [PATCH 2/8] fix: updated docs --- README.md | 6 ++-- docs/source/index.rst | 5 +++- docs/source/installation.rst | 53 ++++++++++++++++++++++++++++++++++-- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 171bf71a..4fd284b8 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ git checkout tags/ -b pip install . # For core API only pip install -e . # For editable install, useful for development work ``` -For more information on how to use the Causal Testing Framework, please refer to our [documentation](https://causal-testing-framework.readthedocs.io/en/latest/?badge=latest). +For more information on how to use the Causal Testing Framework, please refer to our [documentation](https://causal-testing-framework.readthedocs.io/en/latest/?badge=latest). If you have any questions, you can also reach us by [email](citcom-group@sheffield.ac.uk). >[!NOTE] >We recommend you use a 64-bit OS (standard in most modern machines) as we have had reports of the installation crashing on legacy 32-bit Debian systems. @@ -89,12 +89,12 @@ For more information on how to use the Causal Testing Framework, please refer to 2. If you do not already have causal test cases, you can convert your causal DAG to causal tests by running the following command. ``` -python -m causal_testing generate --dag_path $PATH_TO_DAG --output_path $PATH_TO_TESTS +python -m causal_testing generate --dag-path $PATH_TO_DAG --output $PATH_TO_TESTS ``` 3. You can now execute your tests by running the following command. ``` -python -m causal_testing test --dag_path $PATH_TO_DAG --data_paths $PATH_TO_DATA --test_config $PATH_TO_TESTS --output $OUTPUT +python -m causal_testing test --dag-path $PATH_TO_DAG --data-paths $PATH_TO_DATA --test-config $PATH_TO_TESTS --output $OUTPUT ``` The results will be saved for inspection in a JSON file located at `$OUTPUT`. In the future, we hope to add a visualisation tool to assist with this. diff --git a/docs/source/index.rst b/docs/source/index.rst index 2a0ac68c..1fdb2e84 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,6 +19,8 @@ the inputs and outputs of the system under test, supported by mathematical found enable causal inference. Each causal test case targets the causal effect of a specific intervention on the system under test--that is, a deliberate modification to the input configuration expected to produce a corresponding change in one or more outputs. +If you have any questions about our framework, you can also reach us by [email](citcom-group@sheffield.ac.uk). + .. toctree:: :hidden: :caption: Home @@ -72,8 +74,9 @@ a deliberate modification to the input configuration expected to produce a corre CITCoM Homepage Paper - Figshare PyPI + Conda-forge + Figshare DeepWiki .. toctree:: diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 1acab080..74e5ee6a 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -93,6 +93,55 @@ Next Steps * Check out the :doc:`tutorials` to learn how to use the framework. * Read about :doc:`modules/causal_specification` to understand causal specifications and :doc:`modules/causal_testing` for the end-to-end causal testing process. -* Try the command-line interface for quick and simple testing:: +* Run the command for guidance on how to generate your causal tests directly from your input DAG:: - python -m causal_testing test --help \ No newline at end of file + python -m causal_testing generate --help + +* and the command on guidance on how to execute your causal tests:: + + python -m causal_testing test --help + + +Using the CTF on DAFNI +====================== + +The Causal Testing Framework is also available to run on `DAFNI `_, allowing you to generate causal tests and evaluate causal effects from your input data and DAGs without installing the framework locally. This lets you integrate CTF into workflows with other models or datasets easily. + +Data tab +-------- + +- Upload the required input files as a dataset. Typically, this includes: + + - ``dag.dot`` – the directed acyclic graph defining causal relationships between variables. + - ``runtime_data.csv`` – the CSV file containing runtime input data. + - ``causal_tests.json`` – optional; if provided, the framework will run tests directly. Otherwise, tests will be generated automatically. + + **Note:** Input files must remain in the ``data/inputs`` structure; this is required by the workflow. + +Workflow tab +------------- + +- Select the CTF workflow. +- In the Parameter sets section, click **Create**. +- In the page that opens: + + - Select the model in the workflow (typically ``causal-testing-framework``). + - Complete the sections at the bottom: + + - **Parameters:** Set or confirm environment variables from the ``.env`` file (e.g., ``EXECUTION_MODE``, ``CAUSAL_TESTS``, ``CAUSAL_TEST_RESULTS``). These control whether tests are generated or executed, the filenames, estimator, effect type, and other runtime options. + - **Datasets:** Click the icon and select the dataset containing your input files (``dag.dot``, ``runtime_data.csv``, ``causal_tests.json``). All input files will be placed in the required ``data/inputs`` directory when running the workflow. + +- Unselect the model if needed, click **Continue**, and complete any required metadata such as the name of the parameter set. + +Execute the workflow +---------------------- + +- Click **Execute workflow with parameter set**. +- If successful, the workflow will either generate ``causal_tests.json`` (if not provided) or run the tests and create ``causal_test_results.json`` in ``data/outputs``. +- After completion, you can view the results in the **Data tab** as a new output dataset. + +Customisation and chaining +-------------------------- + +- You can create additional workflows to customise input parameters, filenames, or estimators. +- Multiple CTF workflows can also be chained to run sequential analyses or to integrate with other models and datasets, combining results for more complex causal testing scenarios. From bd6c29a924446dc2ae01923225129aa0c4bdb95b Mon Sep 17 00:00:00 2001 From: f-allian Date: Mon, 5 Jan 2026 16:52:50 +0000 Subject: [PATCH 3/8] add: test files --- dafni/data/inputs/causal_tests.json | 79 +++++++++ dafni/data/outputs/causal_test_results.json | 176 ++++++++++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 dafni/data/inputs/causal_tests.json create mode 100644 dafni/data/outputs/causal_test_results.json diff --git a/dafni/data/inputs/causal_tests.json b/dafni/data/inputs/causal_tests.json new file mode 100644 index 00000000..539dd927 --- /dev/null +++ b/dafni/data/inputs/causal_tests.json @@ -0,0 +1,79 @@ +{ + "tests": [ + { + "name": "vaccine --> cum_vaccinations", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "treatment_variable": "vaccine", + "expected_effect": { + "cum_vaccinations": "SomeEffect" + }, + "formula": "cum_vaccinations ~ vaccine", + "skip": false + }, + { + "name": "vaccine --> cum_vaccinated", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "treatment_variable": "vaccine", + "expected_effect": { + "cum_vaccinated": "SomeEffect" + }, + "formula": "cum_vaccinated ~ vaccine", + "skip": false + }, + { + "name": "vaccine --> cum_infections", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "treatment_variable": "vaccine", + "expected_effect": { + "cum_infections": "SomeEffect" + }, + "formula": "cum_infections ~ vaccine", + "skip": false + }, + { + "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "treatment_variable": "cum_vaccinations", + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false + }, + { + "name": "cum_vaccinations _||_ cum_infections | ['vaccine']", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "treatment_variable": "cum_vaccinations", + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false + }, + { + "name": "cum_vaccinated _||_ cum_infections | ['vaccine']", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "treatment_variable": "cum_vaccinated", + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinated + vaccine", + "alpha": 0.05, + "skip": false + } + ] +} \ No newline at end of file diff --git a/dafni/data/outputs/causal_test_results.json b/dafni/data/outputs/causal_test_results.json new file mode 100644 index 00000000..7edf84e5 --- /dev/null +++ b/dafni/data/outputs/causal_test_results.json @@ -0,0 +1,176 @@ +[ + { + "name": "vaccine --> cum_vaccinations", + "estimate_type": "coefficient", + "effect": "direct", + "treatment_variable": "vaccine", + "expected_effect": { + "cum_vaccinations": "SomeEffect" + }, + "formula": "cum_vaccinations ~ vaccine", + "alpha": 0.05, + "skip": false, + "passed": true, + "result": { + "treatment": "vaccine", + "outcome": "cum_vaccinations", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": { + "vaccine": 315785.1333333332 + }, + "ci_low": { + "vaccine": 315339.1666500188 + }, + "ci_high": { + "vaccine": 316231.1000166476 + } + } + }, + { + "name": "vaccine --> cum_vaccinated", + "estimate_type": "coefficient", + "effect": "direct", + "treatment_variable": "vaccine", + "expected_effect": { + "cum_vaccinated": "SomeEffect" + }, + "formula": "cum_vaccinated ~ vaccine", + "alpha": 0.05, + "skip": false, + "passed": true, + "result": { + "treatment": "vaccine", + "outcome": "cum_vaccinated", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": { + "vaccine": 266389.9166666665 + }, + "ci_low": { + "vaccine": 265943.9382101579 + }, + "ci_high": { + "vaccine": 266835.8951231751 + } + } + }, + { + "name": "vaccine --> cum_infections", + "estimate_type": "coefficient", + "effect": "direct", + "treatment_variable": "vaccine", + "expected_effect": { + "cum_infections": "SomeEffect" + }, + "formula": "cum_infections ~ vaccine", + "alpha": 0.05, + "skip": false, + "passed": true, + "result": { + "treatment": "vaccine", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": { + "vaccine": 3332.883333333332 + }, + "ci_low": { + "vaccine": 3274.9650508109467 + }, + "ci_high": { + "vaccine": 3390.801615855717 + } + } + }, + { + "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']", + "estimate_type": "coefficient", + "effect": "direct", + "treatment_variable": "cum_vaccinations", + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false, + "passed": false, + "result": { + "treatment": "cum_vaccinations", + "outcome": "cum_vaccinated", + "adjustment_set": [ + "vaccine" + ], + "effect_measure": "coefficient", + "effect_estimate": { + "cum_vaccinations": 0.9998656401531978 + }, + "ci_low": { + "cum_vaccinations": 0.9929245394500337 + }, + "ci_high": { + "cum_vaccinations": 1.006806740856362 + } + } + }, + { + "name": "cum_vaccinations _||_ cum_infections | ['vaccine']", + "estimate_type": "coefficient", + "effect": "direct", + "treatment_variable": "cum_vaccinations", + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false, + "passed": true, + "result": { + "treatment": "cum_vaccinations", + "outcome": "cum_infections", + "adjustment_set": [ + "vaccine" + ], + "effect_measure": "coefficient", + "effect_estimate": { + "cum_vaccinations": -0.006416682407512808 + }, + "ci_low": { + "cum_vaccinations": -0.056630100838863134 + }, + "ci_high": { + "cum_vaccinations": 0.04379673602383752 + } + } + }, + { + "name": "cum_vaccinated _||_ cum_infections | ['vaccine']", + "estimate_type": "coefficient", + "effect": "direct", + "treatment_variable": "cum_vaccinated", + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinated + vaccine", + "alpha": 0.05, + "skip": false, + "passed": true, + "result": { + "treatment": "cum_vaccinated", + "outcome": "cum_infections", + "adjustment_set": [ + "vaccine" + ], + "effect_measure": "coefficient", + "effect_estimate": { + "cum_vaccinated": -0.00617690058829258 + }, + "ci_low": { + "cum_vaccinated": -0.056393496121197616 + }, + "ci_high": { + "cum_vaccinated": 0.044039694944612455 + } + } + } +] \ No newline at end of file From 0eaa9ca47765031a6abed7a48285a8ab8b6291e3 Mon Sep 17 00:00:00 2001 From: Michael Foster <13611658+jmafoster1@users.noreply.github.com> Date: Tue, 6 Jan 2026 08:17:00 +0000 Subject: [PATCH 4/8] Added full stop --- dafni/model_definition.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dafni/model_definition.yaml b/dafni/model_definition.yaml index 80cf3722..f1af0d91 100644 --- a/dafni/model_definition.yaml +++ b/dafni/model_definition.yaml @@ -90,7 +90,7 @@ spec: - name: IGNORE_CYCLES title: Ignore Cycles - description: If true, cycles in the DAG are ignored. Generation mode only + description: If true, cycles in the DAG are ignored. Generation mode only. type: boolean default: false required: false From 9ce44681723abbd16a3640590ff48c6683efadb5 Mon Sep 17 00:00:00 2001 From: Michael Foster <13611658+jmafoster1@users.noreply.github.com> Date: Tue, 6 Jan 2026 08:18:07 +0000 Subject: [PATCH 5/8] Generation -> generate for consistency --- dafni/model_definition.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dafni/model_definition.yaml b/dafni/model_definition.yaml index f1af0d91..ccd66ba3 100644 --- a/dafni/model_definition.yaml +++ b/dafni/model_definition.yaml @@ -90,7 +90,7 @@ spec: - name: IGNORE_CYCLES title: Ignore Cycles - description: If true, cycles in the DAG are ignored. Generation mode only. + description: If true, cycles in the DAG are ignored. Generate mode only. type: boolean default: false required: false From 6151e8909b85ab69f34d0f27b29c0af2a0e65c32 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Tue, 6 Jan 2026 08:27:40 +0000 Subject: [PATCH 6/8] Fixed email link --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 1fdb2e84..530c4042 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,7 +19,7 @@ the inputs and outputs of the system under test, supported by mathematical found enable causal inference. Each causal test case targets the causal effect of a specific intervention on the system under test--that is, a deliberate modification to the input configuration expected to produce a corresponding change in one or more outputs. -If you have any questions about our framework, you can also reach us by [email](citcom-group@sheffield.ac.uk). +If you have any questions about our framework, you can also reach us by `email `__. .. toctree:: :hidden: From c67cb75623d08e4140cffd0cfd473edec9e5bb25 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Tue, 6 Jan 2026 08:28:11 +0000 Subject: [PATCH 7/8] CI tests alt text --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4fd284b8..4805a6b4 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) -![example workflow](https://github.com/CITCOM-project/CausalTestingFramework/actions/workflows/ci-tests.yaml/badge.svg) +![CI tests](https://github.com/CITCOM-project/CausalTestingFramework/actions/workflows/ci-tests.yaml/badge.svg) [![codecov](https://codecov.io/gh/CITCOM-project/CausalTestingFramework/branch/main/graph/badge.svg?token=04ijFVrb4a)](https://codecov.io/gh/CITCOM-project/CausalTestingFramework) [![Documentation Status](https://readthedocs.org/projects/causal-testing-framework/badge/?version=latest)](https://causal-testing-framework.readthedocs.io/en/latest/?badge=latest) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/CITCOM-project/CausalTestingFramework) @@ -151,4 +151,4 @@ and the software citation should contain the specific Figshare [DOI](https://ord The Causal Testing Framework is supported by the UK's Engineering and Physical Sciences Research Council (EPSRC), with the project name [CITCOM](https://gtr.ukri.org/projects?ref=EP%2FT030526%2F1) - "_Causal Inference for Testing of Computational Models_" -under the grant EP/T030526/1. \ No newline at end of file +under the grant EP/T030526/1. From 38ef8d8e0661ece1b58d486386502163ae1aa018 Mon Sep 17 00:00:00 2001 From: f-allian Date: Fri, 9 Jan 2026 16:56:32 +0000 Subject: [PATCH 8/8] small tweaks --- .github/workflows/publish-to-dafni.yaml | 2 +- dafni/.env | 13 +- dafni/Dockerfile | 29 ++-- .../{ => causal-tests}/causal_tests.json | 0 dafni/data/inputs/{ => dag-data}/dag.dot | 0 .../{ => runtime-data}/runtime_data.csv | 0 dafni/data/outputs/causal_test_results.json | 12 +- dafni/docker-compose.yaml | 3 +- dafni/entrypoint.sh | 134 ++++++++++++++---- dafni/model_definition.yaml | 15 +- 10 files changed, 146 insertions(+), 62 deletions(-) rename dafni/data/inputs/{ => causal-tests}/causal_tests.json (100%) rename dafni/data/inputs/{ => dag-data}/dag.dot (100%) rename dafni/data/inputs/{ => runtime-data}/runtime_data.csv (100%) diff --git a/.github/workflows/publish-to-dafni.yaml b/.github/workflows/publish-to-dafni.yaml index b255c821..9b322f40 100644 --- a/.github/workflows/publish-to-dafni.yaml +++ b/.github/workflows/publish-to-dafni.yaml @@ -50,7 +50,7 @@ jobs: - name: Build the container run: | - docker build -t ctf:${{ env.VERSION }} -f ./dafni/Dockerfile . + docker build --no-cache -t ctf:${{ env.VERSION }} -f ./dafni/Dockerfile . docker save ctf:${{ env.VERSION }} | gzip > ctf-dafni-${{ env.VERSION }}.tar.gz - name: Log into DAFNI diff --git a/dafni/.env b/dafni/.env index 9cd6cbeb..6c89a50e 100644 --- a/dafni/.env +++ b/dafni/.env @@ -1,17 +1,18 @@ #.env -DAG_PATH=./data/inputs/dag.dot -CAUSAL_TESTS=./data/inputs/causal_tests.json -DATA_PATH=./data/inputs/runtime_data.csv -CAUSAL_TEST_RESULTS=./data/outputs/causal_test_results.json -EXECUTION_MODE=auto +# The default file names environment variables for the DAFNI parameters +CAUSAL_TESTS=causal_tests.json +CAUSAL_TEST_RESULTS=causal_test_results.json + +# The default generate and test environment variables for the DAFNI parameters +EXECUTION_MODE=test ESTIMATOR=LinearRegressionEstimator EFFECT_TYPE=direct ESTIMATE_TYPE=coefficient THREADS=0 IGNORE_CYCLES=false VERBOSE=false -QUERY= +QUERY=None ADEQUACY=false BOOTSTRAP_SIZE=100 SILENT=false diff --git a/dafni/Dockerfile b/dafni/Dockerfile index acb6a541..30b9640b 100644 --- a/dafni/Dockerfile +++ b/dafni/Dockerfile @@ -1,30 +1,25 @@ -# Define the Python version neded for CTF FROM python:3.12-slim -## Prevents Python from writing pyc files ENV PYTHONDONTWRITEBYTECODE=1 -# -## Keeps Python from buffering stdout and stderr to avoid the framework -## from crashing without emitting any logs due to buffering + ENV PYTHONUNBUFFERED=1 -# Label maintainer LABEL maintainer="Dr. Neil Walkinshaw - The University of Sheffield" -# Create a folder for the source code/outputs -RUN mkdir -p ./causal_testing -RUN mkdir -p ./data/outputs +WORKDIR / + +# Copy source code +COPY causal_testing /causal_testing + +# Set PYTHONPATH +ENV PYTHONPATH="/causal_testing:${PYTHONPATH}" -# Copy the source code from local root and test files from build into the container -COPY --chown=nobody ./causal_testing ./causal_testing -COPY --chown=nobody ./dafni/data/inputs ./data/inputs +# Install dependencies +RUN pip install --no-cache-dir causal-testing-framework -# Copy your entrypoint script +# Copy entrypoint COPY dafni/entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh -# Install core dependencies using PyPi -RUN pip install causal-testing-framework --no-cache-dir +RUN chmod +x /entrypoint.sh -# Define the entrypoint/commands ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/dafni/data/inputs/causal_tests.json b/dafni/data/inputs/causal-tests/causal_tests.json similarity index 100% rename from dafni/data/inputs/causal_tests.json rename to dafni/data/inputs/causal-tests/causal_tests.json diff --git a/dafni/data/inputs/dag.dot b/dafni/data/inputs/dag-data/dag.dot similarity index 100% rename from dafni/data/inputs/dag.dot rename to dafni/data/inputs/dag-data/dag.dot diff --git a/dafni/data/inputs/runtime_data.csv b/dafni/data/inputs/runtime-data/runtime_data.csv similarity index 100% rename from dafni/data/inputs/runtime_data.csv rename to dafni/data/inputs/runtime-data/runtime_data.csv diff --git a/dafni/data/outputs/causal_test_results.json b/dafni/data/outputs/causal_test_results.json index 7edf84e5..8f2342cf 100644 --- a/dafni/data/outputs/causal_test_results.json +++ b/dafni/data/outputs/causal_test_results.json @@ -103,13 +103,13 @@ ], "effect_measure": "coefficient", "effect_estimate": { - "cum_vaccinations": 0.9998656401531978 + "cum_vaccinations": 0.9998656401531969 }, "ci_low": { - "cum_vaccinations": 0.9929245394500337 + "cum_vaccinations": 0.9929245394500371 }, "ci_high": { - "cum_vaccinations": 1.006806740856362 + "cum_vaccinations": 1.0068067408563566 } } }, @@ -133,13 +133,13 @@ ], "effect_measure": "coefficient", "effect_estimate": { - "cum_vaccinations": -0.006416682407512808 + "cum_vaccinations": -0.006416682407513252 }, "ci_low": { - "cum_vaccinations": -0.056630100838863134 + "cum_vaccinations": -0.05663010083886358 }, "ci_high": { - "cum_vaccinations": 0.04379673602383752 + "cum_vaccinations": 0.043796736023837074 } } }, diff --git a/dafni/docker-compose.yaml b/dafni/docker-compose.yaml index bc18197b..c5642ca5 100644 --- a/dafni/docker-compose.yaml +++ b/dafni/docker-compose.yaml @@ -6,4 +6,5 @@ services: env_file: - .env volumes: - - .:/usr/src/ + - ./data/inputs:/data/inputs:ro + - ./data/outputs:/data/outputs \ No newline at end of file diff --git a/dafni/entrypoint.sh b/dafni/entrypoint.sh index 9fd298e5..502dfea2 100644 --- a/dafni/entrypoint.sh +++ b/dafni/entrypoint.sh @@ -1,14 +1,67 @@ #!/bin/sh set -e -# Auto-detect mode if EXECUTION_MODE=auto +INPUT_DIR="data/inputs" +OUTPUT_DIR="data/outputs" + +# -------------------------- +# DEBUG ENVIRONMENT VARIABLES +# -------------------------- +echo "=== DEBUG: Environment Variables ===" +echo "ADEQUACY: '$ADEQUACY'" +echo "BOOTSTRAP_SIZE: '$BOOTSTRAP_SIZE'" +echo "EXECUTION_MODE: '$EXECUTION_MODE'" +echo "VERBOSE: '$VERBOSE'" +echo "====================================" + +# -------------------------- +# Discover the DAGs first (.dot) +# -------------------------- +DAG_FILES=$(find "$INPUT_DIR/dag-data" -name "*.dot" 2>/dev/null || true) + +if [ -z "$DAG_FILES" ]; then + echo "ERROR: No .dot DAG file found in $INPUT_DIR/dag-data" + exit 1 +fi + +set -- $DAG_FILES +if [ "$#" -ne 1 ]; then + echo "ERROR: Expected exactly one DAG (.dot) file, found $#" + exit 1 +fi + +DAG_PATH="$1" +echo "Using DAG file: $DAG_PATH" + +# -------------------------- +# Discover runtime CSV data (can be multiple) +# -------------------------- +DATA_PATHS=$(find "$INPUT_DIR/runtime-data" -name "*.csv" 2>/dev/null || true) + +if [ -z "$DATA_PATHS" ]; then + echo "ERROR: No runtime CSV files found in $INPUT_DIR/runtime-data" + exit 1 +fi + +echo "Found CSV files: $DATA_PATHS" + +# Causal tests path for checking if it exists in inputs +CAUSAL_TESTS_INPUT_PATH="$INPUT_DIR/causal-tests/$CAUSAL_TESTS" +# Causal tests path for writing (generate mode) +CAUSAL_TESTS_OUTPUT_PATH="$OUTPUT_DIR/$CAUSAL_TESTS" +# Results path (test mode) +CAUSAL_TEST_RESULTS_PATH="$OUTPUT_DIR/$CAUSAL_TEST_RESULTS" + +# -------------------------- +# Auto-detect mode +# -------------------------- if [ "$EXECUTION_MODE" = "auto" ]; then - if [ -f "$CAUSAL_TESTS" ]; then + if [ -d "$INPUT_DIR/causal-tests" ] && [ -f "$CAUSAL_TESTS_INPUT_PATH" ]; then EXECUTION_MODE="test" - echo "Auto mode: causal_tests.json found -> running TEST mode" + echo "Auto mode: causal tests found in inputs -> running TEST mode" else EXECUTION_MODE="generate" - echo "Auto mode: No causal tests found -> running GENERATE mode" + echo "Auto mode: no causal tests found in inputs -> running GENERATE mode" fi else echo "Execution mode explicitly set to: $EXECUTION_MODE" @@ -19,34 +72,65 @@ fi # -------------------------- if [ "$EXECUTION_MODE" = "generate" ]; then echo "Running causal_testing GENERATE..." + echo "Will write causal tests to: $CAUSAL_TESTS_OUTPUT_PATH" + python -m causal_testing generate \ - --dag-path "$DAG_PATH" \ - --output "$CAUSAL_TESTS" \ - --estimator "$ESTIMATOR" \ - --effect-type "$EFFECT_TYPE" \ - --estimate-type "$ESTIMATE_TYPE" \ - $( [ "$IGNORE_CYCLES" = "true" ] && echo "--ignore-cycles" ) \ - --threads "$THREADS" + -D "$DAG_PATH" \ + -o "$CAUSAL_TESTS_OUTPUT_PATH" \ + -e "$ESTIMATOR" \ + -T "$EFFECT_TYPE" \ + -E "$ESTIMATE_TYPE" \ + -t "$THREADS" \ + $([ "$IGNORE_CYCLES" = "true" ] && echo "-i") # -------------------------- # Test mode # -------------------------- elif [ "$EXECUTION_MODE" = "test" ]; then - if [ ! -f "$CAUSAL_TESTS" ]; then - echo "Error: Causal tests file not found at $CAUSAL_TESTS" + if [ ! -f "$CAUSAL_TESTS_INPUT_PATH" ]; then + echo "ERROR: Causal tests file not found at $CAUSAL_TESTS_INPUT_PATH" exit 1 fi echo "Running causal_testing TEST..." - python -m causal_testing test \ - --dag-path "$DAG_PATH" \ - --data-paths "$DATA_PATH" \ - --test-config "$CAUSAL_TESTS" \ - --output "$CAUSAL_TEST_RESULTS" \ - $( [ "$IGNORE_CYCLES" = "true" ] && echo "--ignore-cycles" ) \ - $( [ "$VERBOSE" = "true" ] && echo "--verbose" ) \ - $( [ -n "$QUERY" ] && echo "--query" "$QUERY" ) \ - $( [ "$ADEQUACY" = "true" ] && echo "--adequacy --adequacy-bootstrap-size $BOOTSTRAP_SIZE" ) \ - $( [ "$SILENT" = "true" ] && echo "--silent" ) \ - $( [ "$BATCH_SIZE" != "0" ] && echo "--batch-size $BATCH_SIZE" ) -fi \ No newline at end of file + echo "Using causal tests from: $CAUSAL_TESTS_INPUT_PATH" + + # DEBUG: Show which branch we're taking + echo "=== DEBUG: Adequacy Check ===" + if [ "$ADEQUACY" = "true" ]; then + echo "ADEQUACY is TRUE - will pass -a -b $BOOTSTRAP_SIZE" + else + echo "ADEQUACY is FALSE - will NOT pass -a -b flags" + fi + echo "=============================" + + # Build command with adequacy flags only when ADEQUACY is true + if [ "$ADEQUACY" = "true" ]; then + echo "DEBUG: Executing WITH adequacy flags" + python -m causal_testing test \ + -D "$DAG_PATH" \ + -d $DATA_PATHS \ + -t "$CAUSAL_TESTS_INPUT_PATH" \ + -o "$CAUSAL_TEST_RESULTS_PATH" \ + $([ "$IGNORE_CYCLES" = "true" ] && echo "-i") \ + $([ "$VERBOSE" = "true" ] && echo "-v") \ + $([ -n "$QUERY" ] && [ "$QUERY" != "None" ] && echo "-q '$QUERY'") \ + -a -b $BOOTSTRAP_SIZE \ + $([ "$SILENT" = "true" ] && echo "-s") \ + $([ "$BATCH_SIZE" != "0" ] && echo "--batch-size $BATCH_SIZE") + else + echo "DEBUG: Executing WITHOUT adequacy flags" + python -m causal_testing test \ + -D "$DAG_PATH" \ + -d $DATA_PATHS \ + -t "$CAUSAL_TESTS_INPUT_PATH" \ + -o "$CAUSAL_TEST_RESULTS_PATH" \ + $([ "$IGNORE_CYCLES" = "true" ] && echo "-i") \ + $([ "$VERBOSE" = "true" ] && echo "-v") \ + $([ -n "$QUERY" ] && [ "$QUERY" != "None" ] && echo "-q '$QUERY'") \ + $([ "$SILENT" = "true" ] && echo "-s") \ + $([ "$BATCH_SIZE" != "0" ] && echo "--batch-size $BATCH_SIZE") + fi +fi + +echo "Execution completed successfully" \ No newline at end of file diff --git a/dafni/model_definition.yaml b/dafni/model_definition.yaml index ccd66ba3..b769b7f7 100644 --- a/dafni/model_definition.yaml +++ b/dafni/model_definition.yaml @@ -143,7 +143,7 @@ spec: A .csv file containing the input runtime data to be used default: - 2b7336cd-eb68-4c1f-8f91-26d8969b8cb3 - path: inputs/ + path: inputs/runtime-data required: true - name: DAG data @@ -151,20 +151,23 @@ spec: A .dot file containing the input DAG to be used default: - 74665fdb-43a2-4c51-b81e-d5299b38bf8c - path: inputs/ + path: inputs/dag-data required: true - name: Causal tests description: > A .JSON file containing the input causal tests to be used. This file can also be generated using the input DAG. - default: - - 6f2f7c1f-81b4-4804-8f86-cca304dc7f66 - path: inputs/ + path: inputs/causal-tests required: false outputs: datasets: + - name: causal_tests.json + type: json + description: > + Generated causal tests from the DAG. Created when running in generate mode or auto mode (when no causal tests are provided). + - name: causal_test_results.json type: json description: > - A JSON file containing the output causal test results. \ No newline at end of file + Results from executing causal tests. Created when running in test mode or auto mode (when causal tests are provided). \ No newline at end of file