diff --git a/.gitignore b/.gitignore index e5e362a..da343c4 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ out/ *.pem *.pub /created_scripts/ +/local/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index f59dfa8..a96fd5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ # DLSync Changelog This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.1.0] - 2026-02-10 +### Added +- Added support for AGENTS object type +- Added support for SEMANTIC_VIEWS object type +- Added support for CORTEX_SEARCH_SERVICES object type +- Added support for NOTEBOOKS object type +### Fixed +- Fixed rollback for undeployed declarative scripts +- Moved SESSION_POLICIES, PASSWORD_POLICIES and AUTHENTICATION_POLICIES from account-level to schema-level objects +- Fixed dependency override failure for unknown scripts not in the current changed scripts + ## [3.0.1] - 2026-02-05 ### Fixed - Fixed session context issue when managing account-level objects (databases, schemas). DLSync metadata tables now use fully qualified names to prevent "table does not exist" errors after Snowflake automatically switches session context. diff --git a/README.md b/README.md index b768f84..6d56c00 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Where - Other account-level objects (users, resource monitors, integrations, etc.) - **database_name_*:** is the database name of your project, - **schema_name_*:** are schemas inside the database, -- **object_type:** is type of the object only 1 of the following (VIEWS, FUNCTIONS, PROCEDURES, FILE_FORMATS, TABLES, SEQUENCES, STAGES, STREAMS, TASKS, STREAMLITS, PIPES, ALERTS, DYNAMIC_TABLES, MASKING_POLICIES), +- **object_type:** is type of the object only 1 of the following (VIEWS, FUNCTIONS, PROCEDURES, FILE_FORMATS, TABLES, SEQUENCES, STAGES, STREAMS, TASKS, STREAMLITS, PIPES, ALERTS, DYNAMIC_TABLES, MASKING_POLICIES, NOTEBOOKS, CORTEX_SEARCH_SERVICES, SEMANTIC_VIEWS, AGENTS), - **object_name_*.sql:** are individual database object scripts. - **config.yml:** is a configuration file used to configure DLSync behavior. - **parameter-[profile-*].properties:** is parameter to value map file. This is going to be used by corresponding individual instances of your database. @@ -131,7 +131,7 @@ For example, if you have a view named `SAMPLE_VIEW` in schema `MY_SCHEMA` in dat The structure and content of the scripts differ based on the type of script. This tool categorizes scripts into 2 types: Declarative scripts and Migration scripts. #### 1. Declarative Script -This type of script is used for object types of VIEWS, FUNCTIONS, PROCEDURES, FILE_FORMATS, PIPES, MASKING_POLICIES, STREAMLITS, RESOURCE_MONITORS, NETWORK_POLICIES, SESSION_POLICIES, PASSWORD_POLICIES, AUTHENTICATION_POLICIES, API_INTEGRATIONS, NOTIFICATION_INTEGRATIONS, SECURITY_INTEGRATIONS, STORAGE_INTEGRATIONS, and WAREHOUSES. +This type of script is used for object types of VIEWS, FUNCTIONS, PROCEDURES, FILE_FORMATS, PIPES, MASKING_POLICIES, NOTEBOOKS, CORTEX_SEARCH_SERVICES, SEMANTIC_VIEWS, AGENTS, STREAMLITS, RESOURCE_MONITORS, NETWORK_POLICIES, SESSION_POLICIES, PASSWORD_POLICIES, AUTHENTICATION_POLICIES, API_INTEGRATIONS, NOTIFICATION_INTEGRATIONS, SECURITY_INTEGRATIONS, STORAGE_INTEGRATIONS, and WAREHOUSES. In this type of script, you define the current state (desired state) of the object. When a change is made to the script, DLSync replaces the current object with the updated definition. These types of scripts must always have a `create or replace` statement. Every time you make a change to the script, DLSync will replace the object with the new definition. @@ -450,7 +450,7 @@ The role must have privileges to create and manage DLSync tracking tables in the > **Note:** the database and schema specified in the connection will be used to store the DLSync tracking tables. ### Database-Level Objects -For managing database-level objects (VIEWS, FUNCTIONS, PROCEDURES, TABLES, SEQUENCES, STAGES, STREAMS, TASKS, ALERTS, DYNAMIC_TABLES, FILE_FORMATS, PIPES, MASKING_POLICIES), the role must have: +For managing database-level objects (VIEWS, FUNCTIONS, PROCEDURES, TABLES, SEQUENCES, STAGES, STREAMS, TASKS, ALERTS, DYNAMIC_TABLES, FILE_FORMATS, PIPES, MASKING_POLICIES, SESSION_POLICIES, PASSWORD_POLICIES, AUTHENTICATION_POLICIES, NOTEBOOKS), the role must have: - **USAGE** on the target schema - **CREATE** privileges for the specific object types being deployed (CREATE VIEW, CREATE FUNCTION, CREATE TABLE, etc.) - **ALTER** privileges on existing objects in the schema @@ -465,9 +465,8 @@ For managing account-level objects, the role must have: - **CREATE INTEGRATION** (for SECURITY_INTEGRATIONS, STORAGE_INTEGRATIONS, NOTIFICATION_INTEGRATIONS, API_INTEGRATIONS) - **CREATE RESOURCE MONITOR** (for RESOURCE_MONITORS) - **CREATE NETWORK POLICY** (for NETWORK_POLICIES) -- **CREATE SESSION POLICY** (for SESSION_POLICIES) -- **CREATE PASSWORD POLICY** (for PASSWORD_POLICIES) -- **CREATE AUTHENTICATION POLICY** (for AUTHENTICATION_POLICIES) + +> **Note:** SESSION_POLICIES, PASSWORD_POLICIES, and AUTHENTICATION_POLICIES are schema-level objects (created within a schema) but can be applied to accounts or users via `ALTER ACCOUNT SET` or `ALTER USER SET`. ## DLSync Metadata Tables DLSync stores script metadata, deployment history, and logs in the database. diff --git a/build.gradle b/build.gradle index 8b95002..6d2ca55 100644 --- a/build.gradle +++ b/build.gradle @@ -17,8 +17,8 @@ dependencies { implementation 'org.apache.commons:commons-text:1.14.0' implementation 'net.snowflake:snowflake-jdbc:3.25.1' - implementation 'ch.qos.logback:logback-core:1.5.18' - implementation 'ch.qos.logback:logback-classic:1.5.18' + implementation 'ch.qos.logback:logback-core:1.5.25' + implementation 'ch.qos.logback:logback-classic:1.5.25' implementation 'org.slf4j:slf4j-api:2.0.4' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.18.2' implementation 'commons-cli:commons-cli:1.9.0' diff --git a/example_scripts/main/example_db/MAIN_SCHEMA/AGENTS/SALES_ASSISTANT.sql b/example_scripts/main/example_db/MAIN_SCHEMA/AGENTS/SALES_ASSISTANT.sql new file mode 100644 index 0000000..53f891b --- /dev/null +++ b/example_scripts/main/example_db/MAIN_SCHEMA/AGENTS/SALES_ASSISTANT.sql @@ -0,0 +1,40 @@ +CREATE OR REPLACE AGENT ${EXAMPLE_DB}.${MAIN_SCHEMA}.SALES_ASSISTANT + COMMENT = 'Sales assistant agent for product and order queries' + PROFILE = '{"display_name": "Sales Assistant", "avatar": "sales-icon.png", "color": "blue"}' + FROM SPECIFICATION + $$ + models: + orchestration: claude-4-sonnet + + orchestration: + budget: + seconds: 30 + tokens: 16000 + + instructions: + response: "You will respond in a friendly but professional manner about sales and products" + orchestration: "For product questions use Search; for sales analytics use Analyst" + system: "You are a helpful sales assistant that helps with product and order inquiries" + sample_questions: + - question: "What products do we have in stock?" + answer: "I'll search our product catalog to find available items." + - question: "How many orders were placed today?" + answer: "I'll analyze our sales data to find the order count." + + tools: + - tool_spec: + type: "cortex_analyst_text_to_sql" + name: "SalesAnalyst" + description: "Analyzes sales data and generates reports" + - tool_spec: + type: "cortex_search" + name: "ProductSearch" + description: "Searches product catalog" + + tool_resources: + SalesAnalyst: + semantic_view: "${EXAMPLE_DB}.${MAIN_SCHEMA}.SALES_ANALYTICS" + ProductSearch: + name: "${EXAMPLE_DB}.${MAIN_SCHEMA}.PRODUCT_SEARCH" + max_results: "10" + $$; diff --git a/example_scripts/main/ACCOUNT/AUTHENTICATION_POLICIES/MFA_POLICY.SQL b/example_scripts/main/example_db/MAIN_SCHEMA/AUTHENTICATION_POLICIES/MFA_POLICY.SQL similarity index 66% rename from example_scripts/main/ACCOUNT/AUTHENTICATION_POLICIES/MFA_POLICY.SQL rename to example_scripts/main/example_db/MAIN_SCHEMA/AUTHENTICATION_POLICIES/MFA_POLICY.SQL index 330eb0d..cd9dddd 100644 --- a/example_scripts/main/ACCOUNT/AUTHENTICATION_POLICIES/MFA_POLICY.SQL +++ b/example_scripts/main/example_db/MAIN_SCHEMA/AUTHENTICATION_POLICIES/MFA_POLICY.SQL @@ -1,4 +1,4 @@ -CREATE OR REPLACE AUTHENTICATION POLICY mfa_policy +CREATE OR REPLACE AUTHENTICATION POLICY ${EXAMPLE_DB}.${MAIN_SCHEMA}.MFA_POLICY AUTHENTICATION_METHODS = ('PASSWORD', 'KEYPAIR') MFA_ENROLLMENT = 'REQUIRED' CLIENT_TYPES = ('ALL') diff --git a/example_scripts/main/example_db/MAIN_SCHEMA/CORTEX_SEARCH_SERVICES/PRODUCT_SEARCH.sql b/example_scripts/main/example_db/MAIN_SCHEMA/CORTEX_SEARCH_SERVICES/PRODUCT_SEARCH.sql new file mode 100644 index 0000000..4660c27 --- /dev/null +++ b/example_scripts/main/example_db/MAIN_SCHEMA/CORTEX_SEARCH_SERVICES/PRODUCT_SEARCH.sql @@ -0,0 +1,12 @@ +CREATE OR REPLACE CORTEX SEARCH SERVICE ${EXAMPLE_DB}.${MAIN_SCHEMA}.PRODUCT_SEARCH + ON PRODUCT_NAME + ATTRIBUTES PRICE, STOCK + WAREHOUSE = ${MY_WAREHOUSE} + TARGET_LAG = '1 hour' +AS ( + SELECT + PRODUCT_NAME, + PRICE, + STOCK + FROM ${EXAMPLE_DB}.${MAIN_SCHEMA}.PRODUCTS +); diff --git a/example_scripts/main/example_db/MAIN_SCHEMA/NOTEBOOKS/SALES_ANALYSIS_NOTEBOOK.SQL b/example_scripts/main/example_db/MAIN_SCHEMA/NOTEBOOKS/SALES_ANALYSIS_NOTEBOOK.SQL new file mode 100644 index 0000000..096dd49 --- /dev/null +++ b/example_scripts/main/example_db/MAIN_SCHEMA/NOTEBOOKS/SALES_ANALYSIS_NOTEBOOK.SQL @@ -0,0 +1,5 @@ +CREATE OR REPLACE NOTEBOOK ${EXAMPLE_DB}.${MAIN_SCHEMA}.SALES_ANALYSIS_NOTEBOOK + MAIN_FILE='sales_analysis.ipynb' + QUERY_WAREHOUSE='${MY_WAREHOUSE}' + COMMENT = 'Example notebook deployed via DLSync' +; diff --git a/example_scripts/main/ACCOUNT/PASSWORD_POLICIES/PASSWORD_STRENGTH_POLICY.SQL b/example_scripts/main/example_db/MAIN_SCHEMA/PASSWORD_POLICIES/PASSWORD_STRENGTH_POLICY.SQL similarity index 75% rename from example_scripts/main/ACCOUNT/PASSWORD_POLICIES/PASSWORD_STRENGTH_POLICY.SQL rename to example_scripts/main/example_db/MAIN_SCHEMA/PASSWORD_POLICIES/PASSWORD_STRENGTH_POLICY.SQL index 5702989..a4623a3 100644 --- a/example_scripts/main/ACCOUNT/PASSWORD_POLICIES/PASSWORD_STRENGTH_POLICY.SQL +++ b/example_scripts/main/example_db/MAIN_SCHEMA/PASSWORD_POLICIES/PASSWORD_STRENGTH_POLICY.SQL @@ -1,4 +1,4 @@ -CREATE OR REPLACE PASSWORD POLICY password_strength_policy +CREATE OR REPLACE PASSWORD POLICY ${EXAMPLE_DB}.${MAIN_SCHEMA}.PASSWORD_STRENGTH_POLICY PASSWORD_MIN_LENGTH = 12 PASSWORD_MAX_LENGTH = 256 PASSWORD_MIN_UPPER_CASE_CHARS = 1 diff --git a/example_scripts/main/example_db/MAIN_SCHEMA/SEMANTIC_VIEWS/SALES_ANALYTICS.sql b/example_scripts/main/example_db/MAIN_SCHEMA/SEMANTIC_VIEWS/SALES_ANALYTICS.sql new file mode 100644 index 0000000..c21de60 --- /dev/null +++ b/example_scripts/main/example_db/MAIN_SCHEMA/SEMANTIC_VIEWS/SALES_ANALYTICS.sql @@ -0,0 +1,29 @@ +CREATE OR REPLACE SEMANTIC VIEW ${EXAMPLE_DB}.${MAIN_SCHEMA}.SALES_ANALYTICS + TABLES ( + products AS ${EXAMPLE_DB}.${MAIN_SCHEMA}.PRODUCTS + PRIMARY KEY (ID) + COMMENT = 'Product catalog table', + orders AS ${EXAMPLE_DB}.${MAIN_SCHEMA}.ORDERS + PRIMARY KEY (ID) + COMMENT = 'Customer orders table' + ) + RELATIONSHIPS ( + orders (PRODUCT_ID) REFERENCES products + ) + DIMENSIONS ( + products.product_name AS products.PRODUCT_NAME + WITH SYNONYMS = ('product', 'item') + COMMENT = 'Name of the product', + orders.order_date AS orders.ORDER_DATE + WITH SYNONYMS = ('date', 'purchase date') + COMMENT = 'Date when order was placed' + ) + METRICS ( + orders.total_quantity AS SUM(orders.QUANTITY) + WITH SYNONYMS = ('quantity', 'units sold') + COMMENT = 'Total quantity ordered', + orders.order_count AS COUNT(orders.ID) + WITH SYNONYMS = ('number of orders', 'count') + COMMENT = 'Total number of orders' + ) + COMMENT = 'Semantic view for sales analytics'; diff --git a/example_scripts/main/ACCOUNT/SESSION_POLICIES/SESSION_TIMEOUT_POLICY.SQL b/example_scripts/main/example_db/MAIN_SCHEMA/SESSION_POLICIES/SESSION_TIMEOUT_POLICY.SQL similarity index 58% rename from example_scripts/main/ACCOUNT/SESSION_POLICIES/SESSION_TIMEOUT_POLICY.SQL rename to example_scripts/main/example_db/MAIN_SCHEMA/SESSION_POLICIES/SESSION_TIMEOUT_POLICY.SQL index d221931..09dc62d 100644 --- a/example_scripts/main/ACCOUNT/SESSION_POLICIES/SESSION_TIMEOUT_POLICY.SQL +++ b/example_scripts/main/example_db/MAIN_SCHEMA/SESSION_POLICIES/SESSION_TIMEOUT_POLICY.SQL @@ -1,4 +1,4 @@ -CREATE OR REPLACE SESSION POLICY session_timeout_policy +CREATE OR REPLACE SESSION POLICY ${EXAMPLE_DB}.${MAIN_SCHEMA}.SESSION_TIMEOUT_POLICY SESSION_IDLE_TIMEOUT_MINS = 60 SESSION_UI_IDLE_TIMEOUT_MINS = 15 COMMENT = 'Session timeout policy for security'; diff --git a/gradle.properties b/gradle.properties index 0f2f246..efbd9d0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -releaseVersion=3.0.1 \ No newline at end of file +releaseVersion=3.1.0 \ No newline at end of file diff --git a/pom.xml b/pom.xml index 9c5992b..c193297 100644 --- a/pom.xml +++ b/pom.xml @@ -27,13 +27,13 @@ ch.qos.logback logback-core - 1.5.18 + 1.5.25 ch.qos.logback logback-classic - 1.5.18 + 1.5.25 org.slf4j diff --git a/src/main/java/com/snowflake/dlsync/ChangeManager.java b/src/main/java/com/snowflake/dlsync/ChangeManager.java index e0cd743..33a7284 100644 --- a/src/main/java/com/snowflake/dlsync/ChangeManager.java +++ b/src/main/java/com/snowflake/dlsync/ChangeManager.java @@ -98,6 +98,7 @@ public void rollback() throws SQLException, IOException { .stream() .filter(script -> !config.isScriptExcluded(script)) .filter(script -> !script.getObjectType().isMigration()) + .filter(script -> scriptRepo.isScriptPreviouslyDeployed(script)) .filter(script -> scriptRepo.isScriptChanged(script)) .collect(Collectors.toList()); dependencyGraph.addNodes(changedScripts); diff --git a/src/main/java/com/snowflake/dlsync/dependency/DependencyGraph.java b/src/main/java/com/snowflake/dlsync/dependency/DependencyGraph.java index 74e7e86..26fc397 100644 --- a/src/main/java/com/snowflake/dlsync/dependency/DependencyGraph.java +++ b/src/main/java/com/snowflake/dlsync/dependency/DependencyGraph.java @@ -108,12 +108,14 @@ public List