Add Jupyter notebook for instance data visualization and update data generation process

Author: markcoletti

examples-proposed/024-aggregated-compute-ensemble/gen_data.py

@@ -1,6 +1,14 @@
 #!/usr/bin/env python3
 """
     Used to generate synthetic data as an example.
+
+    Writes two output files:
+
+    * `<instance ID>_solution.json`: contains the synthetic data
+    * `<instance ID>_stats.csv`: contains provenance information about the run
+
+    The JSON file is, in turn, read by the a per-instance jupyter notebook
+    available on the Portal to generate a plot of the data.
 """
 import argparse
 from typing import Any
@@ -9,7 +17,6 @@
 from time import time
 from traceback import print_exc
 import numpy as np
-import matplotlib.pyplot as plt
 
 from ipsframework.resourceHelper import get_platform_info
 from ipsframework.services import add_analysis_data_file
@@ -58,13 +65,7 @@ def main(instance: str,
             u_new[i] = u[i] + r * (u[i + 1] - 2 * u[i] + u[i - 1])
         u = u_new
 
-    # Plotting the result
-    plt.plot(x, u)
-    plt.xlabel("Position (x)")
-    plt.ylabel("Temperature (u)")
-    plt.title("Solution of 1D Heat Equation")
-    plt.grid(True)
-    plt.savefig(f"{instance}_solution.png")
+
 
     # Save some per-component stats
     stats_fname = f'{instance}_stats.csv'

examples-proposed/024-aggregated-compute-ensemble/instance_base_notebook.ipynb

@@ -0,0 +1,101 @@
+{
+ "cells": [
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": [
+    "# Instance base notebook\n",
+    "\n",
+    "This notebook replicates for each ensemble instance."
+   ],
+   "id": "56c3997f94a366d5"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-12-18T18:45:35.472364Z",
+     "start_time": "2025-12-18T18:45:35.462785Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import json\n",
+    "from pathlib import Path\n",
+    "\n",
+    "%matplotlib notebook\n",
+    "import matplotlib.pyplot as plt"
+   ],
+   "id": "937aff95abd359fa",
+   "outputs": [],
+   "execution_count": 5
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-12-18T18:40:41.874866Z",
+     "start_time": "2025-12-18T18:40:41.864743Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "# Find the JSON output from `gen_data.py` that was invoked as a sub-task. It will\n",
+    "# follow the naming pattern `<instance ID>_solution.json`.  It will contain two\n",
+    "# arrays, `x` and `u`, which we will plot.\n",
+    "solution_json_file = list(Path(\".\").rglob(\"*solution.json\"))\n",
+    "\n",
+    "if len(solution_json) != 1:\n",
+    "    raise RuntimeError(\n",
+    "        f\"Expected exactly one solution JSON file, found {len(solution_json_file)}\"\n",
+    "    )\n",
+    "solution_json_file = solution_json_file[0]\n",
+    "\n",
+    "# Now read the JSON file and extract the data.\n",
+    "with open(solution_json_file, \"r\") as f:\n",
+    "    solution_data = json.load(f)\n",
+    "\n",
+    "x = solution_data[\"x\"]\n",
+    "u = solution_data[\"u\"]\n",
+    "\n",
+    "# Now plot the data.\n",
+    "plt.plot(x, u)\n",
+    "plt.xlabel(\"Position (x)\")\n",
+    "plt.ylabel(\"Temperature (u)\")\n",
+    "plt.title(\"Solution of 1D Heat Equation\")\n",
+    "plt.grid(True)\n",
+    "plt.savefig(f\"{instance}_solution.png\")"
+   ],
+   "id": "b1a40da330ba6b3e",
+   "outputs": [],
+   "execution_count": 1
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "66002c188c5ab473"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples-proposed/024-aggregated-compute-ensemble/instance_component.py

@@ -71,12 +71,12 @@ def step(self, timestamp: float = 0.0, **keywords):
         self.services.info(f'{instance_id}: Completed MPI executable with '
                            f'return value: {return_value}.')
 
-        # TODO temporarily commenting this out until the actual
-        # example is ready to consider adding data files to the portal.  This
-        # originally came from code Lance wrote in a previous example.
-        # try:
-        #     self.services.add_analysis_data_files([data_fname, stats_fname], timestamp)
-        # except Exception:
-        #     print('did not add data files to portal, check logs')
+        # Add the generated data JSON and CSV files to the portal
+        try:
+            self.services.add_analysis_data_files([f'{instance_id}_solution.json',
+                                                   f'{instance_id}_stats.csv'],
+                                                  replace=True)
+        except Exception:
+            print('did not add data files to portal, check logs')
 
         self.services.info(f'{instance_id}: End of step of instance component.')

examples-proposed/024-aggregated-compute-ensemble/instance_driver.py

@@ -5,11 +5,22 @@
 
 from ipsframework import Component
 
+# The notebook that will be copied for each ensemble instance
+SOURCE_NOTEBOOK_NAME='instance_base_notebook.ipynb'
+
 
 class InstanceDriver(Component):
     """
     Instance driver component that steps the main component
     """
+    def init(self, timestamp: float = 0.0, **keywords):
+        self.services.stage_input_files([SOURCE_NOTEBOOK_NAME])
+
+        self.services.initialize_jupyter_notebook(
+                dest_notebook_name='jupyterhub_instance_notebook.ipynb',
+                source_notebook_path=SOURCE_NOTEBOOK_NAME,
+        )
+
 
     def step(self, timestamp: float = 0.0, **keywords):
         instance_component = self.services.get_port('WORKER')