diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 47c3fccf29..8de29895ba 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,3 +6,12 @@ updates: interval: daily labels: - dependencies + + # Copy of above config for formplayer + - package-ecosystem: gradle + directory: "/" + target-branch: "formplayer" + schedule: + interval: daily + labels: + - dependencies diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c475f5f12c..91683bfe79 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest env: JAVA_VERSION: '17' - JOB_GRADLE_VERSION: 7.3 + JOB_GRADLE_VERSION: 8.1 FORMPLAYER_GRADLE_VERSION: 8.1 FORMPLAYER_JAVA_VERSION: '17' steps: diff --git a/build.gradle b/build.gradle index 87077cdbce..7a15ece772 100644 --- a/build.gradle +++ b/build.gradle @@ -14,6 +14,7 @@ plugins { // Needed even though it looks unused sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 configurations { ccapi.extendsFrom(ccapiImplementation) @@ -74,7 +75,7 @@ dependencies { implementation 'commons-cli:commons-cli:1.3.1' implementation 'joda-time:joda-time:2.12.1' implementation 'com.carrotsearch:hppc:0.9.1' - api 'com.squareup.retrofit2:retrofit:2.3.0' + api 'com.squareup.retrofit2:retrofit:2.9.0' api 'com.squareup.okhttp3:okhttp:4.11.0' implementation 'com.google.code.findbugs:jsr305:3.0.2' implementation 'io.reactivex.rxjava2:rxjava:2.2.21' @@ -88,7 +89,7 @@ dependencies { cliImplementation 'commons-cli:commons-cli:1.3.1' cliImplementation 'joda-time:joda-time:2.9.4' cliImplementation 'io.reactivex.rxjava2:rxjava:2.1.1' - cliImplementation 'com.squareup.retrofit2:retrofit:2.3.0' + cliImplementation 'com.squareup.retrofit2:retrofit:2.9.0' cliImplementation 'com.squareup.okhttp3:okhttp:4.11.0' cliImplementation 'com.google.guava:guava:31.1-jre' cliImplementation 'com.datadoghq:dd-trace-api:1.10.0' @@ -101,7 +102,7 @@ dependencies { testImplementation 'org.json:json:20250517' testImplementation 'org.xerial:sqlite-jdbc:3.40.0.0' testImplementation 'joda-time:joda-time:2.9.4' - testImplementation 'com.squareup.retrofit2:retrofit:2.3.0' + testImplementation 'com.squareup.retrofit2:retrofit:2.9.0' testImplementation 'com.squareup.okhttp3:okhttp:4.11.0' testImplementation 'com.google.guava:guava:31.1-jre' @@ -112,7 +113,7 @@ dependencies { ccapiImplementation 'joda-time:joda-time:2.9.4' ccapiImplementation 'com.carrotsearch:hppc:0.9.1' ccapiImplementation 'io.reactivex.rxjava2:rxjava:2.1.1' - ccapiImplementation "com.squareup.retrofit2:retrofit:2.3.0" + ccapiImplementation "com.squareup.retrofit2:retrofit:2.9.0" ccapiImplementation 'com.squareup.okhttp3:okhttp:4.11.0' ccapiImplementation 'com.google.guava:guava:31.1-jre' ccapiImplementation 'io.opentracing:opentracing-api:0.33.0' @@ -126,7 +127,7 @@ dependencies { translateImplementation 'joda-time:joda-time:2.9.4' translateImplementation 'org.json:json:20250517' translateImplementation 'io.reactivex.rxjava2:rxjava:2.1.1' - translateImplementation "com.squareup.retrofit2:retrofit:2.3.0" + translateImplementation "com.squareup.retrofit2:retrofit:2.9.0" translateImplementation 'com.squareup.okhttp3:okhttp:4.11.0' translateImplementation 'com.google.guava:guava:31.1-jre' diff --git a/src/cli/java/org/commcare/util/cli/CliMain.java b/src/cli/java/org/commcare/util/cli/CliMain.java index d2e4605aa8..8ad51ae546 100644 --- a/src/cli/java/org/commcare/util/cli/CliMain.java +++ b/src/cli/java/org/commcare/util/cli/CliMain.java @@ -61,7 +61,13 @@ public static void main(String[] args) { return; } - cliCommand.handle(); + try { + cliCommand.handle(); + } catch (Exception e) { + System.err.println("\nCommand failed with error:"); + e.printStackTrace(); + System.exit(-1); + } } private static CliCommand getCliCommand(String commandName) throws CliCommandNotFound { diff --git a/src/cli/java/org/commcare/util/screen/EntityScreen.java b/src/cli/java/org/commcare/util/screen/EntityScreen.java index a4333c7666..4ab0c989d8 100755 --- a/src/cli/java/org/commcare/util/screen/EntityScreen.java +++ b/src/cli/java/org/commcare/util/screen/EntityScreen.java @@ -188,6 +188,7 @@ public boolean initReferences(SessionWrapper session) throws CommCareSessionExce } } } + initialized = true; return true; } diff --git a/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java b/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java index 1b2df52bc8..24b9186aab 100644 --- a/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java +++ b/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java @@ -1,11 +1,9 @@ package org.commcare.util.screen; -import org.commcare.cases.entity.AsyncEntity; import org.commcare.cases.entity.AsyncNodeEntityFactory; import org.commcare.cases.entity.Entity; import org.commcare.cases.entity.EntitySortNotificationInterface; import org.commcare.cases.entity.EntitySorter; -import org.commcare.cases.entity.EntityStorageCache; import org.commcare.cases.entity.EntityStringFilterer; import org.commcare.cases.entity.NodeEntityFactory; import org.commcare.suite.model.Detail; @@ -16,7 +14,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.Hashtable; import java.util.List; import java.util.Map; @@ -38,7 +35,7 @@ public static List> initEntities(EvaluationContext context EntityScreenContext entityScreenContext, TreeReference[] entitiesRefs) { NodeEntityFactory nodeEntityFactory; if (detail.isLazyLoading()) { - nodeEntityFactory = new AsyncNodeEntityFactory(detail, context, null); + nodeEntityFactory = new AsyncNodeEntityFactory(detail, context, null, false); } else { nodeEntityFactory = new NodeEntityFactory(detail, context); } diff --git a/src/main/java/org/commcare/cases/entity/AsyncEntity.java b/src/main/java/org/commcare/cases/entity/AsyncEntity.java index 2d9f521e26..e97e556cc8 100755 --- a/src/main/java/org/commcare/cases/entity/AsyncEntity.java +++ b/src/main/java/org/commcare/cases/entity/AsyncEntity.java @@ -1,7 +1,11 @@ package org.commcare.cases.entity; +import static org.commcare.cases.entity.EntityStorageCache.ValueType.TYPE_NORMAL_FIELD; +import static org.commcare.cases.entity.EntityStorageCache.ValueType.TYPE_SORT_FIELD; + import org.commcare.cases.util.StringUtils; +import org.commcare.suite.model.Detail; import org.commcare.suite.model.DetailField; import org.commcare.suite.model.DetailGroup; import org.commcare.suite.model.Text; @@ -13,8 +17,6 @@ import org.javarosa.xpath.expr.XPathExpression; import org.javarosa.xpath.parser.XPathSyntaxException; -import java.io.Closeable; -import java.io.IOException; import java.util.Enumeration; import java.util.Hashtable; @@ -39,12 +41,14 @@ public class AsyncEntity extends Entity { private final DetailField[] fields; private final Object[] data; private final String[] sortData; - private final boolean[] relevancyData; + private final Boolean[] relevancyData; private final String[][] sortDataPieces; + private final String[] altTextData; private final EvaluationContext context; private final Hashtable mVariableDeclarations; private final DetailGroup mDetailGroup; + private final boolean cacheEnabled; private boolean mVariableContextLoaded = false; private final String mCacheIndex; private final String mDetailId; @@ -53,29 +57,19 @@ public class AsyncEntity extends Entity { private final EntityStorageCache mEntityStorageCache; /* - * the Object's lock. NOTE: _DO NOT LOCK ANY CODE WHICH READS/WRITES THE CACHE - * UNTIL YOU HAVE A LOCK FOR THE DB! - * - * The lock is for the integrity of this object, not the larger environment, - * and any DB access has its own implict lock between threads, so it's easy - * to accidentally deadlock if you don't already have the db lock - * - * Basically you should never be calling mEntityStorageCache from inside of - * a lock that + * the Object's lock for the integrity of this object */ private final Object mAsyncLock = new Object(); - public AsyncEntity(DetailField[] fields, EvaluationContext ec, - TreeReference t, Hashtable variables, - @Nullable EntityStorageCache cache, String cacheIndex, String detailId, - String extraKey, DetailGroup detailGroup) { + public AsyncEntity(Detail detail, EvaluationContext ec, + TreeReference t, Hashtable variables, + @Nullable EntityStorageCache cache, String cacheIndex, String extraKey) { super(t, extraKey); - - this.fields = fields; + this.fields = detail.getFields(); this.data = new Object[fields.length]; this.sortData = new String[fields.length]; this.sortDataPieces = new String[fields.length][]; - this.relevancyData = new boolean[fields.length]; + this.relevancyData = new Boolean[fields.length]; this.altTextData = new String[fields.length]; this.context = ec; this.mVariableDeclarations = variables; @@ -85,8 +79,9 @@ public AsyncEntity(DetailField[] fields, EvaluationContext ec, //if this ref is _cachable_ every time, since it's a pretty big lift this.mCacheIndex = cacheIndex; - this.mDetailId = detailId; - this.mDetailGroup = detailGroup; + this.mDetailId = detail.getId(); + this.mDetailGroup = detail.getGroup(); + this.cacheEnabled = detail.isCacheEnabled(); } private void loadVariableContext() { @@ -106,18 +101,43 @@ private void loadVariableContext() { @Override public Object getField(int i) { synchronized (mAsyncLock) { - loadVariableContext(); + if (data[i] != null) { + return data[i]; + } + if (!fields[i].isCacheEnabled()) { + data[i] = evaluateField(i); + return data[i]; + } + String cacheKey = null; if (data[i] == null) { - try { - data[i] = fields[i].getTemplate().evaluate(context); - } catch (XPathException xpe) { - Logger.exception("Error while evaluating field for case list ", xpe); - xpe.printStackTrace(); - data[i] = ""; + if (mEntityStorageCache != null && mCacheIndex != null) { + cacheKey = mEntityStorageCache.getCacheKey(mDetailId, String.valueOf(i), + TYPE_NORMAL_FIELD); + // Return from the cache if we have a value + String value = mEntityStorageCache.retrieveCacheValue(mCacheIndex, cacheKey); + if (value != null) { + data[i] = value; + return data[i]; + } } } - return data[i]; + data[i] = evaluateField(i); + if (mEntityStorageCache != null && mCacheIndex != null) { + mEntityStorageCache.cache(mCacheIndex, cacheKey, String.valueOf(data[i])); + } } + return data[i]; + } + + private Object evaluateField(int i) { + loadVariableContext(); + try { + data[i] = fields[i].getTemplate().evaluate(context); + } catch (XPathException xpe) { + Logger.exception("Error while evaluating field for case list ", xpe); + data[i] = ""; + } + return data[i]; } @Override @@ -131,51 +151,63 @@ public String getNormalizedField(int i) { @Override public String getSortField(int i) { - try (Closeable ignored = mEntityStorageCache != null ? mEntityStorageCache.lockCache() : null) { - //get our second lock. - synchronized (mAsyncLock) { - if (sortData[i] == null) { - // sort data not in search field cache; load and store it - Text sortText = fields[i].getSort(); - if (sortText == null) { - return null; + synchronized (mAsyncLock) { + if (sortData[i] != null) { + return sortData[i]; + } + + // eval and return if field is not marked as optimize + if (cacheEnabled && !fields[i].isCacheEnabled()) { + evaluateSortData(i); + return sortData[i]; + } + + String cacheKey = null; + if (sortData[i] == null) { + Text sortText = fields[i].getSort(); + if (sortText == null) { + return null; + } + + if (mEntityStorageCache != null) { + if (cacheEnabled) { + cacheKey = mEntityStorageCache.getCacheKey(mDetailId, String.valueOf(i), + TYPE_SORT_FIELD); + } else { + // old cache and index + cacheKey = i + "_" + TYPE_SORT_FIELD; } - String cacheKey = null; - if (mEntityStorageCache != null) { - cacheKey = mEntityStorageCache.getCacheKey(mDetailId, String.valueOf(i)); - if (mCacheIndex != null) { - //Check the cache! - String value = mEntityStorageCache.retrieveCacheValue(mCacheIndex, cacheKey); - if (value != null) { - this.setSortData(i, value); - return sortData[i]; - } - - loadVariableContext(); - try { - sortText = fields[i].getSort(); - if (sortText == null) { - this.setSortData(i, getFieldString(i)); - } else { - this.setSortData(i, StringUtils.normalize(sortText.evaluate(context))); - } - if (mEntityStorageCache != null) { - mEntityStorageCache.cache(mCacheIndex, cacheKey, sortData[i]); - } - } catch (XPathException xpe) { - Logger.exception("Error while evaluating sort field", xpe); - xpe.printStackTrace(); - sortData[i] = ""; - } + if (mCacheIndex != null) { + //Check the cache! + String value = mEntityStorageCache.retrieveCacheValue(mCacheIndex, cacheKey); + if (value != null) { + this.setSortData(i, value); + return sortData[i]; } } } - return sortData[i]; } - } catch (IOException e) { - Logger.exception("Error while getting sort field", e); + evaluateSortData(i); + if (mEntityStorageCache != null && mCacheIndex != null) { + mEntityStorageCache.cache(mCacheIndex, cacheKey, sortData[i]); + } + return sortData[i]; + } + } + + private void evaluateSortData(int i) { + loadVariableContext(); + try { + Text sortText = fields[i].getSort(); + if (sortText == null) { + this.setSortData(i, getFieldString(i)); + } else { + this.setSortData(i, StringUtils.normalize(sortText.evaluate(context))); + } + } catch (XPathException xpe) { + Logger.exception("Error while evaluating sort field", xpe); + sortData[i] = ""; } - return null; } @Override @@ -193,15 +225,24 @@ public boolean isValidField(int fieldIndex) { if (getField(fieldIndex).equals("")) { return false; } + return getRelevancyData(fieldIndex); + } + } + private boolean getRelevancyData(int i) { + synchronized (mAsyncLock) { + if (relevancyData[i] != null) { + return relevancyData[i]; + } + loadVariableContext(); try { - this.relevancyData[fieldIndex] = this.fields[fieldIndex].isRelevant(this.context); + relevancyData[i] = fields[i].isRelevant(context); } catch (XPathSyntaxException e) { - final String msg = "Invalid relevant condition for field : " + fields[fieldIndex].getHeader().toString(); + final String msg = "Invalid relevant condition for field : " + fields[i].getHeader().toString(); Logger.exception(msg, e); - throw new RuntimeException(msg); + throw new XPathException(e); } - return this.relevancyData[fieldIndex]; + return relevancyData[i]; } } @@ -228,13 +269,29 @@ private void setSortData(int i, String val) { } } + private void setFieldData(int i, String val) { + synchronized (mAsyncLock) { + data[i] = val; + } + } + public void setSortData(String cacheKey, String val) { if (mEntityStorageCache == null) { throw new IllegalStateException("No entity cache defined"); } - int sortIndex = mEntityStorageCache.getSortFieldIdFromCacheKey(mDetailId, cacheKey); - if (sortIndex != -1) { - setSortData(sortIndex, val); + int fieldIndex = mEntityStorageCache.getFieldIdFromCacheKey(mDetailId, cacheKey); + if (fieldIndex != -1) { + setSortData(fieldIndex, val); + } + } + + public void setFieldData(String cacheKey, String val) { + if (mEntityStorageCache == null) { + throw new IllegalStateException("No entity cache defined"); + } + int fieldIndex = mEntityStorageCache.getFieldIdFromCacheKey(mDetailId, cacheKey); + if (fieldIndex != -1) { + setFieldData(fieldIndex, val); } } @@ -252,7 +309,7 @@ private static String[] breakUpField(String input) { @Override public String getGroupKey() { if (mDetailGroup != null) { - return (String)mDetailGroup.getFunction().eval(context); + return (String)mDetailGroup.getFunction().eval(context); } return null; } @@ -260,6 +317,9 @@ public String getGroupKey() { @Nullable public String getAltTextData(int i) { synchronized (mAsyncLock) { + if (altTextData[i] != null) { + return altTextData[i]; + } loadVariableContext(); Text altText = fields[i].getAltText(); if (altText != null) { @@ -267,7 +327,6 @@ public String getAltTextData(int i) { altTextData[i] = altText.evaluate(context); } catch (XPathException xpe) { Logger.exception("Error while evaluating field for case list ", xpe); - xpe.printStackTrace(); altTextData[i] = ""; } } diff --git a/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java b/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java index 6f2821ed9e..dac78b721f 100755 --- a/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java +++ b/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java @@ -1,7 +1,10 @@ package org.commcare.cases.entity; +import static org.commcare.cases.entity.EntityLoadingProgressListener.EntityLoadingProgressPhase.PHASE_UNCACHED_CALCULATION; + import org.commcare.suite.model.Detail; +import org.commcare.suite.model.DetailField; import org.javarosa.core.model.condition.EvaluationContext; import org.javarosa.core.model.instance.TreeReference; import org.javarosa.core.model.utils.CacheHost; @@ -12,12 +15,10 @@ import java.util.List; import javax.annotation.Nullable; - /** * @author ctsims */ public class AsyncNodeEntityFactory extends NodeEntityFactory { - private static final String TAG = AsyncNodeEntityFactory.class.getSimpleName(); private final OrderedHashtable mVariableDeclarations; private final Hashtable mEntitySet = new Hashtable<>(); @@ -32,13 +33,19 @@ public class AsyncNodeEntityFactory extends NodeEntityFactory { // Don't show entity list until we primeCache and caches all fields private final boolean isBlockingAsyncMode; + /** + * Whether we are loading entity in a background process. + * Used to accelerate processing in foreground by skipping lazy properties + */ + private boolean inBackground; + public AsyncNodeEntityFactory(Detail d, EvaluationContext ec, - @Nullable EntityStorageCache entityStorageCache) { + @Nullable EntityStorageCache entityStorageCache, boolean inBackground) { super(d, ec); - mVariableDeclarations = detail.getVariableDeclarations(); mEntityCache = entityStorageCache; isBlockingAsyncMode = detail.hasSortField(); + this.inBackground = inBackground; } @Override @@ -57,8 +64,8 @@ public Entity getEntity(TreeReference data) { String entityKey = loadCalloutDataMapKey(nodeContext); AsyncEntity entity = - new AsyncEntity(detail.getFields(), nodeContext, data, mVariableDeclarations, - mEntityCache, mCacheIndex, detail.getId(), entityKey, detail.getGroup()); + new AsyncEntity(detail, nodeContext, data, mVariableDeclarations, + mEntityCache, mCacheIndex, entityKey); if (mCacheIndex != null) { mEntitySet.put(mCacheIndex, entity); @@ -80,7 +87,8 @@ protected void setEvaluationContextDefaultQuerySet(EvaluationContext ec, * Bulk loads search field cache from db. * Note that the cache is lazily built upon first case list search. */ - private void primeCache() { + protected void primeCache() { + if (isCancelled) return; if (mEntityCache == null || mTemplateIsCachable == null || !mTemplateIsCachable || mCacheHost == null) { return; } @@ -89,15 +97,25 @@ private void primeCache() { if (cachePrimeKeys == null) { return; } + updateProgress(EntityLoadingProgressListener.EntityLoadingProgressPhase.PHASE_CACHING, 0, 100); mEntityCache.primeCache(mEntitySet,cachePrimeKeys, detail); + updateProgress(EntityLoadingProgressListener.EntityLoadingProgressPhase.PHASE_CACHING, 100, 100); + } + + private void updateProgress(EntityLoadingProgressListener.EntityLoadingProgressPhase phase, int progress, + int total) { + if (progressListener != null) { + progressListener.publishEntityLoadingProgress(phase, progress, total); + } } @Override protected void prepareEntitiesInternal(List> entities) { + // Legacy cache and index code, only here to maintain backward compatibility // if blocking mode load cache on the same thread and set any data thats not cached if (isBlockingAsyncMode) { primeCache(); - setUnCachedData(entities); + setUnCachedDataOld(entities); } else { // otherwise we want to show the entity list asap and hence want to offload the loading cache part to a separate // thread while caching any uncached data later on UI thread during Adapter's getView @@ -110,12 +128,57 @@ protected void prepareEntitiesInternal(List> entities) { } } - private void setUnCachedData(List> entities) { + @Override + public void cacheEntities(List> entities) { + if (detail.isCacheEnabled()) { + primeCache(); + setUnCachedData(entities); + } else { + primeCache(); + setUnCachedDataOld(entities); + } + } + + protected void setUnCachedData(List> entities) { + boolean foregroundWithLazyLoading = !inBackground && detail.isLazyLoading(); + boolean foregroundWithoutLazyLoading = !inBackground && !detail.isLazyLoading(); + for (int i = 0; i < entities.size(); i++) { + if (isCancelled) return; + AsyncEntity e = (AsyncEntity)entities.get(i); + for (int col = 0; col < e.getNumFields(); ++col) { + DetailField field = detail.getFields()[col]; + /** + * 1. If we are in foreground with lazy loading turned on, the priority is to show + * the user screen asap. Therefore, we need to skip calculating lazy fields. + * 2. If we are in foreground with lazy loading turned off, we want to calculate all fields here. + * 3. If we are in background with lazy loading turned on or off, we want to calculate all fields + * backed by cache in order to keep them ready for when user loads the list. + */ + if (foregroundWithoutLazyLoading || (foregroundWithLazyLoading && !field.isLazyLoading()) || ( + inBackground && field.isCacheEnabled())) { + e.getField(col); + if (field.getSort() != null) { + e.getSortField(col); + } + } + } + if (i % 100 == 0) { + updateProgress(PHASE_UNCACHED_CALCULATION, i, entities.size()); + } + } + updateProgress(PHASE_UNCACHED_CALCULATION, entities.size(), entities.size()); + } + + // Old cache and index pathway where we only cache sort fields + @Deprecated + protected void setUnCachedDataOld(List> entities) { for (int i = 0; i < entities.size(); i++) { + if (isCancelled) return; AsyncEntity e = (AsyncEntity)entities.get(i); for (int col = 0; col < e.getNumFields(); ++col) { - e.getSortField(col); + e.getSortField(col); } + updateProgress(PHASE_UNCACHED_CALCULATION, i, entities.size()); } } diff --git a/src/main/java/org/commcare/cases/entity/EntityLoadingProgressListener.java b/src/main/java/org/commcare/cases/entity/EntityLoadingProgressListener.java new file mode 100644 index 0000000000..3886452193 --- /dev/null +++ b/src/main/java/org/commcare/cases/entity/EntityLoadingProgressListener.java @@ -0,0 +1,57 @@ +package org.commcare.cases.entity; + +/** + * Listener to track progress of loading an entity list + */ +public interface EntityLoadingProgressListener { + + enum EntityLoadingProgressPhase { + + /** + * Phase in which we build entity models from entity references, includes case detail calculations + * normally except when we are lazy loading entities. + * This is the only relevant phase for normal (non cache and index) case lists. + */ + PHASE_PROCESSING(1), + + /** + * Only relevant when entity cache is enabled, involves loading the entity cache into memory + */ + PHASE_CACHING(2), + + + /** + * Phase in which we calculate any uncached entity fields. Can take much longer if things are not + * already cached and similarly can be very quick when most things are available in cache. + */ + PHASE_UNCACHED_CALCULATION(3); + + private final int value; + + private EntityLoadingProgressPhase(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + + public static EntityLoadingProgressPhase fromInt(int i) { + for (EntityLoadingProgressPhase phase : EntityLoadingProgressPhase.values()) { + if (phase.getValue() == i) { + return phase; + } + } + throw new IllegalArgumentException("Unexpected value: " + i); + } + } + + /** + * Method to implement to listen to the entity loading progress + * + * @param phase The specific phase of entity loading process + * @param progress progress corresponding to the current entity loading phase + * @param total max progress corresponding to the current entity loading phase + */ + void publishEntityLoadingProgress(EntityLoadingProgressPhase phase, int progress, int total); +} diff --git a/src/main/java/org/commcare/cases/entity/EntityStorageCache.java b/src/main/java/org/commcare/cases/entity/EntityStorageCache.java index 589d3c0895..69ea8163ac 100644 --- a/src/main/java/org/commcare/cases/entity/EntityStorageCache.java +++ b/src/main/java/org/commcare/cases/entity/EntityStorageCache.java @@ -9,15 +9,20 @@ * Interface for evaluated entity fields cache */ public interface EntityStorageCache { + enum ValueType { + TYPE_NORMAL_FIELD, + TYPE_SORT_FIELD + } + Closeable lockCache(); - String getCacheKey(String detailId, String detailFieldIndex); + String getCacheKey(String detailId, String detailFieldIndex, ValueType valueType); String retrieveCacheValue(String cacheIndex, String cacheKey); void cache(String cacheIndex, String cacheKey, String data); - int getSortFieldIdFromCacheKey(String detailId, String cacheKey); + int getFieldIdFromCacheKey(String detailId, String cacheKey); void primeCache(Hashtable entitySet, String[][] cachePrimeKeys, Detail detail); } diff --git a/src/main/java/org/commcare/cases/entity/NodeEntityFactory.java b/src/main/java/org/commcare/cases/entity/NodeEntityFactory.java index 28e2793127..9e25eb81ca 100755 --- a/src/main/java/org/commcare/cases/entity/NodeEntityFactory.java +++ b/src/main/java/org/commcare/cases/entity/NodeEntityFactory.java @@ -25,6 +25,13 @@ public class NodeEntityFactory { protected final EvaluationContext ec; protected final Detail detail; private ReducingTraceReporter traceReporter; + protected EntityLoadingProgressListener progressListener; + + /** + * Flag that denotes cancellation of the underlying process responsible for loading entities + * Implementations of long running methods can check this to interrupt and exit early + */ + protected volatile boolean isCancelled = false; public NodeEntityFactory(Detail d, EvaluationContext ec) { this.detail = d; @@ -149,6 +156,7 @@ protected void setEvaluationContextDefaultQuerySet(EvaluationContext ec, * Performs the underlying work to prepare the entity set * (see prepareEntities()). Separated out to enforce timing * related to preparing and utilizing results + * * @param entities */ protected void prepareEntitiesInternal(List> entities) { @@ -192,4 +200,32 @@ public final boolean isEntitySetReady() { return isEntitySetReadyInternal(); } } + + /** + * Caches the provided entities. Default implementation throws RuntimeException. + * Subclasses should override this method if they support caching. + * + * @param entities List of entities to cache + * @throws RuntimeException if caching is not supported + */ + public void cacheEntities(List> entities) { + throw new RuntimeException("Method not supported for normal Node Entity Factory"); + } + + /** + * Sets the progress listener for entity loading operations. + * + * @param progressListener The progress listener to use + */ + public void setEntityProgressListener(EntityLoadingProgressListener progressListener) { + if (this.progressListener != null) { + throw new RuntimeException( + "Entity loading progress listener is already set in entity factory"); + } + this.progressListener = progressListener; + } + + public void markAsCancelled() { + isCancelled = true; + } } diff --git a/src/main/java/org/commcare/core/interfaces/HttpResponseProcessor.java b/src/main/java/org/commcare/core/interfaces/HttpResponseProcessor.java index a922464635..11ac9b420b 100644 --- a/src/main/java/org/commcare/core/interfaces/HttpResponseProcessor.java +++ b/src/main/java/org/commcare/core/interfaces/HttpResponseProcessor.java @@ -12,7 +12,7 @@ public interface HttpResponseProcessor { /** * Http response was in the 200s */ - void processSuccess(int responseCode, InputStream responseData); + void processSuccess(int responseCode, InputStream responseData, String apiVersion); /** * Http response was in the 400s. @@ -20,7 +20,7 @@ public interface HttpResponseProcessor { * Can represent authentication issues, data parity issues between client * and server, among other things */ - void processClientError(int responseCode); + void processClientError(int responseCode, InputStream errorStream); /** * Http response was in the 500s diff --git a/src/main/java/org/commcare/core/interfaces/ResponseStreamAccessor.java b/src/main/java/org/commcare/core/interfaces/ResponseStreamAccessor.java index 2309b441f0..e61ad466c4 100644 --- a/src/main/java/org/commcare/core/interfaces/ResponseStreamAccessor.java +++ b/src/main/java/org/commcare/core/interfaces/ResponseStreamAccessor.java @@ -3,6 +3,12 @@ import java.io.IOException; import java.io.InputStream; +import javax.annotation.Nullable; + public interface ResponseStreamAccessor { InputStream getResponseStream() throws IOException; + + @Nullable + InputStream getErrorResponseStream() throws IOException; + String getApiVersion(); } diff --git a/src/main/java/org/commcare/core/network/AuthInfo.java b/src/main/java/org/commcare/core/network/AuthInfo.java index 7e874eb40e..f2fbe590f2 100644 --- a/src/main/java/org/commcare/core/network/AuthInfo.java +++ b/src/main/java/org/commcare/core/network/AuthInfo.java @@ -1,5 +1,7 @@ package org.commcare.core.network; +import org.commcare.cases.util.StringUtils; + /** * Created by amstone326 on 5/8/18. */ @@ -10,6 +12,8 @@ public abstract class AuthInfo { public String password; public boolean wrapDomain; + public String bearerToken; + public static class NoAuth extends AuthInfo { } @@ -20,11 +24,16 @@ public ProvidedAuth(String username, String password) { } public ProvidedAuth(String username, String password, boolean wrapDomain) { + if (StringUtils.isEmpty(username)) { + throw new IllegalArgumentException("ProvidedAuth requires a non-empty username"); + } + if (StringUtils.isEmpty(password)) { + throw new IllegalArgumentException("ProvidedAuth requires a non-empty password"); + } this.username = username; this.password = password; this.wrapDomain = wrapDomain; } - } // Auth with the currently-logged in user @@ -32,4 +41,12 @@ public static class CurrentAuth extends AuthInfo { } + public static class TokenAuth extends AuthInfo { + public TokenAuth(String token) { + if (StringUtils.isEmpty(token)) { + throw new IllegalArgumentException("TokenAuth requires a non-empty token"); + } + bearerToken = token; + } + } } diff --git a/src/main/java/org/commcare/core/network/ModernHttpRequester.java b/src/main/java/org/commcare/core/network/ModernHttpRequester.java index 66106cb08c..92e5b3c46b 100644 --- a/src/main/java/org/commcare/core/network/ModernHttpRequester.java +++ b/src/main/java/org/commcare/core/network/ModernHttpRequester.java @@ -8,18 +8,19 @@ import org.commcare.core.network.bitcache.BitCacheFactory; import org.commcare.util.NetworkStatus; import org.javarosa.core.io.StreamsUtil; +import org.javarosa.core.services.Logger; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; -import javax.net.ssl.SSLHandshakeException; -import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.SSLException; import okhttp3.FormBody; import okhttp3.MultipartBody; @@ -34,7 +35,7 @@ * * @author Phillip Mates (pmates@dimagi.com) */ -public class ModernHttpRequester implements ResponseStreamAccessor { +public class ModernHttpRequester { /** * How long to wait when opening network connection in milliseconds */ @@ -88,7 +89,29 @@ public void makeRequestAndProcess() { } try { response = makeRequest(); - processResponse(responseProcessor, response.code(), this); + final ModernHttpRequester requester = this; + processResponse(responseProcessor, response.code(), new ResponseStreamAccessor() { + /** + * Only gets called if response processor is supplied + * @return Input Stream from cache + * @throws IOException if an io error happens while reading or writing to cache + */ + @Override + public InputStream getResponseStream() throws IOException { + return requester.getResponseStream(response); + } + + @Nullable + @Override + public InputStream getErrorResponseStream() throws IOException { + return requester.getErrorResponseStream(response); + } + + @Override + public String getApiVersion() { + return requester.getApiVersion(); + } + }); } catch (IOException e) { e.printStackTrace(); responseProcessor.handleIOException(e); @@ -122,14 +145,22 @@ public Response makeRequest() throws IOException { private Response executeAndCheckCaptivePortals(Call currentCall) throws IOException { try { return currentCall.execute(); - } catch (SSLHandshakeException | SSLPeerUnverifiedException e) { + } catch (SSLException e) { + // SSLHandshakeException is thrown by the CommcareRequestGenerator on + // 4.3 devices when the peer certificate is bad. + // + // SSLPeerUnverifiedException is thrown by the CommcareRequestGenerator + // on 2.3 devices when the peer certificate is bad. + // // This may be a real SSL exception associated with the real endpoint server, or this // might be a property of the local network. + if(NetworkStatus.isCaptivePortal()) { throw new CaptivePortalRedirectException(); } //Otherwise just rethrow the original exception. Probably a certificate issue + //Could be related to local clock issue throw e; } } @@ -146,12 +177,21 @@ public static void processResponse(HttpResponseProcessor responseProcessor, responseProcessor.handleIOException(e); return; } - responseProcessor.processSuccess(responseCode, responseStream); + String apiVersion = streamAccessor.getApiVersion(); + responseProcessor.processSuccess(responseCode, responseStream, apiVersion); } finally { StreamsUtil.closeStream(responseStream); } } else if (responseCode >= 400 && responseCode < 500) { - responseProcessor.processClientError(responseCode); + InputStream errorStream = null; + try { + errorStream = streamAccessor.getErrorResponseStream(); + responseProcessor.processClientError(responseCode, errorStream); + } catch (Exception e) { + Logger.exception("Exception during network error processing", e); + } finally { + StreamsUtil.closeStream(errorStream); + } } else if (responseCode >= 500 && responseCode < 600) { responseProcessor.processServerError(responseCode); } else { @@ -167,6 +207,11 @@ public static void processResponse(HttpResponseProcessor responseProcessor, public InputStream getResponseStream(Response response) throws IOException { InputStream inputStream = response.body().byteStream(); + return cacheResponse(inputStream, response); + } + + private InputStream cacheResponse(InputStream inputStream, Response response) + throws IOException { BitCache cache = BitCacheFactory.getCache(cacheDirSetup, getContentLength(response)); cache.initializeCache(); OutputStream cacheOut = cache.getCacheStream(); @@ -174,14 +219,16 @@ public InputStream getResponseStream(Response response) throws IOE return cache.retrieveCache(); } - /** - * Only gets called if response processor is supplied - * @return Input Stream from cache - * @throws IOException if an io error happens while reading or writing to cache - */ - @Override - public InputStream getResponseStream() throws IOException { - return getResponseStream(response); + @Nullable + public InputStream getErrorResponseStream(Response response) throws IOException { + if (response.errorBody() != null) { + return cacheResponse( response.errorBody().byteStream(), response); + } + return null; + } + + public String getApiVersion() { + return response != null ? response.headers().get("x-api-current-version") : null; } public static RequestBody getPostBody(Multimap inputs) { diff --git a/src/main/java/org/commcare/modern/reference/JavaHttpReference.java b/src/main/java/org/commcare/modern/reference/JavaHttpReference.java index 081c8e10d3..1bae94b460 100644 --- a/src/main/java/org/commcare/modern/reference/JavaHttpReference.java +++ b/src/main/java/org/commcare/modern/reference/JavaHttpReference.java @@ -10,8 +10,7 @@ import java.net.HttpURLConnection; import java.net.URL; -import javax.net.ssl.SSLHandshakeException; -import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.SSLException; /** * @author ctsims @@ -44,7 +43,7 @@ public InputStream getStream() throws IOException { HttpURLConnection.setFollowRedirects(true); return conn.getInputStream(); - } catch (SSLHandshakeException | SSLPeerUnverifiedException e) { + } catch (SSLException e) { if(NetworkStatus.isCaptivePortal()) { throw new CaptivePortalRedirectException(); } diff --git a/src/main/java/org/commcare/modern/session/SessionWrapper.java b/src/main/java/org/commcare/modern/session/SessionWrapper.java index 53c205da03..8573766da2 100644 --- a/src/main/java/org/commcare/modern/session/SessionWrapper.java +++ b/src/main/java/org/commcare/modern/session/SessionWrapper.java @@ -41,6 +41,10 @@ public SessionWrapper(CommCareSession session, CommCarePlatform platform, UserSa public SessionWrapper(CommCareSession session, CommCarePlatform platform, UserSandbox sandbox, String windowWidth) { this(session, platform, sandbox, null, windowWidth); } + + public SessionWrapper(CommCareSession session, CommCarePlatform platform, UserSandbox sandbox) { + this(session, platform, sandbox, null, null); + } public SessionWrapper(CommCarePlatform platform, UserSandbox sandbox) { super(platform); diff --git a/src/main/java/org/commcare/session/SessionInstanceBuilder.java b/src/main/java/org/commcare/session/SessionInstanceBuilder.java index a638f88114..95bcead922 100644 --- a/src/main/java/org/commcare/session/SessionInstanceBuilder.java +++ b/src/main/java/org/commcare/session/SessionInstanceBuilder.java @@ -24,7 +24,6 @@ public static TreeElement getSessionInstance(SessionFrame frame, String deviceId addSessionNavData(sessionRoot, frame); addMetadata(sessionRoot, deviceId, appversion, username, userId, drift, windowWidth, applanguage); addUserProperties(sessionRoot, userFields); - return sessionRoot; } @@ -100,7 +99,6 @@ private static void addMetadata(TreeElement sessionRoot, String deviceId, addData(sessionMeta, "drift", String.valueOf(drift)); addData(sessionMeta, "window_width", windowWidth); addData(sessionMeta, "applanguage", applanguage); - sessionRoot.addChild(sessionMeta); } diff --git a/src/main/java/org/commcare/suite/model/Detail.java b/src/main/java/org/commcare/suite/model/Detail.java index 05aa585d1a..95be8e2f43 100755 --- a/src/main/java/org/commcare/suite/model/Detail.java +++ b/src/main/java/org/commcare/suite/model/Detail.java @@ -109,6 +109,7 @@ public class Detail implements Externalizable { // Loads detail fields lazily when required private boolean lazyLoading; + private boolean cacheEnabled; private DetailGroup group; @@ -126,7 +127,7 @@ public Detail(String id, DisplayUnit title, Text noItemsText, String nodeset, Ve Vector actions, Callout callout, String fitAcross, String uniformUnitsString, String forceLandscape, String focusFunction, String printPathProvided, String relevancy, Global global, DetailGroup group, - boolean lazyLoading, Text selectText) { + boolean lazyLoading,boolean cacheEnabled, Text selectText) { if (detailsVector.size() > 0 && fieldsVector.size() > 0) { throw new IllegalArgumentException("A detail may contain either sub-details or fields, but not both."); @@ -178,6 +179,7 @@ public Detail(String id, DisplayUnit title, Text noItemsText, String nodeset, Ve this.global = global; this.group = group; this.lazyLoading = lazyLoading; + this.cacheEnabled = cacheEnabled; } /** @@ -233,6 +235,10 @@ public boolean isLazyLoading() { return lazyLoading; } + public boolean isCacheEnabled() { + return cacheEnabled; + } + /** * Given a detail, return an array of details that will contain either * - all child details @@ -263,7 +269,11 @@ public boolean isCompound() { * Whether this detail is expected to be so huge in scope that * the platform should limit its strategy for loading it to be asynchronous * and cached on special keys. + * + * Legacy way to turn on cache and index: can be removed once we migrate to new + * cache and index entirely */ + @Deprecated public boolean useAsyncStrategy() { for (DetailField f : getFields()) { if (f.getSortOrder() == DetailField.SORT_ORDER_CACHABLE) { @@ -298,6 +308,7 @@ public void readExternal(DataInputStream in, PrototypeFactory pf) throws IOExcep group = (DetailGroup) ExtUtil.read(in, new ExtWrapNullable(DetailGroup.class), pf); lazyLoading = ExtUtil.readBool(in); selectText = (Text) ExtUtil.read(in, new ExtWrapNullable(Text.class), pf); + cacheEnabled = ExtUtil.readBool(in); } @Override @@ -322,6 +333,7 @@ public void writeExternal(DataOutputStream out) throws IOException { ExtUtil.write(out, new ExtWrapNullable(group)); ExtUtil.writeBool(out, lazyLoading); ExtUtil.write(out, new ExtWrapNullable(selectText)); + ExtUtil.writeBool(out, cacheEnabled); } public OrderedHashtable getVariableDeclarations() { @@ -503,6 +515,18 @@ public boolean hasSortField() { return false; } + // Returns true if we should trigger any optimizations for this detail + public boolean shouldOptimize() { + if (cacheEnabled || lazyLoading) { + for (DetailField field : fields) { + if (field.isCacheEnabled() || field.isLazyLoading()) { + return true; + } + } + } + return false; + } + private abstract class Map { private final E a; diff --git a/src/main/java/org/commcare/suite/model/DetailField.java b/src/main/java/org/commcare/suite/model/DetailField.java index 82bdb77572..b8010710d4 100644 --- a/src/main/java/org/commcare/suite/model/DetailField.java +++ b/src/main/java/org/commcare/suite/model/DetailField.java @@ -77,6 +77,9 @@ public class DetailField implements Externalizable { private String fontSize; private String cssID; + private boolean lazyLoading; + private boolean cacheEnabled; + public DetailField() { } @@ -214,6 +217,8 @@ public void readExternal(DataInputStream in, PrototypeFactory pf) throws IOExcep endpointAction = (EndpointAction)ExtUtil.read(in, new ExtWrapNullable(EndpointAction.class), pf); showBorder = ExtUtil.readBool(in); showShading = ExtUtil.readBool(in); + cacheEnabled = ExtUtil.readBool(in); + lazyLoading = ExtUtil.readBool(in); } @Override @@ -247,6 +252,8 @@ public void writeExternal(DataOutputStream out) throws IOException { ExtUtil.write(out, new ExtWrapNullable(endpointAction)); ExtUtil.writeBool(out, showBorder); ExtUtil.writeBool(out, showShading); + ExtUtil.writeBool(out, cacheEnabled); + ExtUtil.writeBool(out, lazyLoading); } public int getGridX() { @@ -299,6 +306,14 @@ public boolean getShowShading() { return showShading; } + public boolean isLazyLoading() { + return lazyLoading; + } + + public boolean isCacheEnabled() { + return cacheEnabled; + } + public static class Builder { final DetailField field; @@ -434,5 +449,13 @@ public void setShowBorder(boolean showBorder) { } public void setShowShading(boolean showShading) { field.showShading = showShading; } + + public void setCacheEnabled(boolean cacheEnabled) { + field.cacheEnabled = cacheEnabled; + } + + public void setLazyLoading(boolean lazyLoading) { + field.lazyLoading = lazyLoading; + } } } diff --git a/src/main/java/org/commcare/util/CommCarePlatform.java b/src/main/java/org/commcare/util/CommCarePlatform.java index fae9888858..9099d23a32 100644 --- a/src/main/java/org/commcare/util/CommCarePlatform.java +++ b/src/main/java/org/commcare/util/CommCarePlatform.java @@ -4,11 +4,13 @@ import org.commcare.resources.model.ResourceTable; import org.commcare.suite.model.Detail; import org.commcare.suite.model.Endpoint; +import org.commcare.suite.model.EntityDatum; import org.commcare.suite.model.Entry; import org.commcare.suite.model.FormEntry; import org.commcare.suite.model.Menu; import org.commcare.suite.model.OfflineUserRestore; import org.commcare.suite.model.Profile; +import org.commcare.suite.model.SessionDatum; import org.commcare.suite.model.Suite; import org.javarosa.core.model.instance.FormInstance; import org.javarosa.core.services.PropertyManager; @@ -20,6 +22,7 @@ import java.util.Enumeration; import java.util.Hashtable; import java.util.List; +import java.util.Map; import java.util.Vector; import javax.annotation.Nullable; @@ -215,6 +218,33 @@ public String getMenuDisplayStyle(String menuId) { return commonDisplayStyle; } + + /** + * Loops through complete set of detail config and checks whether + * any of them are cache enabled + * + * @return true if entity caching is enabled for any of the detail configs in app, false otherwise + */ + public boolean isEntityCachingEnabled() { + Map commandMap = getCommandToEntryMap(); + for (String command : commandMap.keySet()) { + Entry entry = commandMap.get(command); + if (entry == null) continue; + for (SessionDatum sessionDatum : entry.getSessionDataReqs()) { + if (sessionDatum instanceof EntityDatum) { + String shortDetailId = ((EntityDatum)sessionDatum).getShortDetail(); + if (shortDetailId != null) { + Detail detail = getDetail(shortDetailId); + if (detail.isCacheEnabled()) { + return true; + } + } + } + } + } + return false; + } + public OfflineUserRestore getDemoUserRestore() { return offlineUserRestore; } diff --git a/src/main/java/org/commcare/util/EncryptionUtils.java b/src/main/java/org/commcare/util/EncryptionUtils.java index cb317cb313..4731470494 100644 --- a/src/main/java/org/commcare/util/EncryptionUtils.java +++ b/src/main/java/org/commcare/util/EncryptionUtils.java @@ -4,6 +4,8 @@ import java.nio.charset.Charset; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; +import java.security.Key; +import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import javax.crypto.BadPaddingException; @@ -18,43 +20,85 @@ public class EncryptionUtils { /** - * Encrypts a message using the AES encryption and produces a base64 encoded payload containing the ciphertext, and a random IV which was used to encrypt the input. + * Encrypts a message and produces a base64 encoded payload containing the ciphertext + * The key and transform are specified as inputs to + * A random IV may be generated to encrypt the input (unless using RSA) * - * @param message a UTF-8 encoded message to be encrypted + * @param message a byte[] to be encrypted * @param key A base64 encoded 256 bit symmetric key + * @param transform The transformation to use for encryption + * @param includeMessageLength Whether to include the message length in the packed payload * @return A base64 encoded payload containing the IV and AES encrypted ciphertext, which can be decoded by this utility's decrypt method and the same symmetric key */ - public static String encrypt(String message, String key) throws EncryptionException { - final String ENCRYPT_ALGO = "AES/GCM/NoPadding"; + public static String encrypt(byte[] message, Key key, String transform, + boolean includeMessageLength) throws EncryptionException { final int MIN_IV_LENGTH_BYTE = 1; final int MAX_IV_LENGTH_BYTE = 255; - SecretKey secret = getSecretKeySpec(key); + + if(key == null) { + throw new NullPointerException("Key is null"); + } + + if(transform == null) { + throw new NullPointerException("Transform is null"); + } + + boolean allowEmptyIV = transform.startsWith("RSA"); try { - Cipher cipher = Cipher.getInstance(ENCRYPT_ALGO); - cipher.init(Cipher.ENCRYPT_MODE, secret); - byte[] encryptedMessage = cipher.doFinal(message.getBytes(Charset.forName("UTF-8"))); + Cipher cipher = Cipher.getInstance(transform); + cipher.init(Cipher.ENCRYPT_MODE, key); + byte[] encryptedMessage = cipher.doFinal(message); byte[] iv = cipher.getIV(); - if (iv.length < MIN_IV_LENGTH_BYTE || iv.length > MAX_IV_LENGTH_BYTE) { + int ivLength = iv == null ? 0 : iv.length; + if (!allowEmptyIV && (ivLength < MIN_IV_LENGTH_BYTE || ivLength > MAX_IV_LENGTH_BYTE)) { throw new EncryptionException("Initialization vector should be between " + MIN_IV_LENGTH_BYTE + " and " + MAX_IV_LENGTH_BYTE + - " bytes long, but it is " + iv.length + " bytes"); + " bytes long, but it is " + ivLength + " bytes"); } + + int extraBytes = includeMessageLength ? 2 : 0; + // The conversion of iv.length to byte takes the low 8 bits. To // convert back, cast to int and mask with 0xFF. - byte[] ivPlusMessage = ByteBuffer.allocate(1 + iv.length + encryptedMessage.length) - .put((byte)iv.length) - .put(iv) - .put(encryptedMessage) - .array(); - return Base64.encode(ivPlusMessage); - } catch (Exception ex) { - throw new EncryptionException("Unknown error during encryption", ex); + ByteBuffer byteBuffer = ByteBuffer.allocate(1 + ivLength + extraBytes + encryptedMessage.length) + .put((byte) ivLength); + + if(iv != null) { + byteBuffer.put(iv); + } + + if(includeMessageLength) { + byteBuffer.put((byte)(encryptedMessage.length / 256)); + byteBuffer.put((byte)(encryptedMessage.length % 256)); + } + + byteBuffer.put(encryptedMessage); + + return Base64.encode(byteBuffer.array()); + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | + BadPaddingException | IllegalBlockSizeException e) { + throw new EncryptionException("Error during encryption", e); } } + /** + * Encrypts a message using the AES encryption and produces a base64 encoded payload containing the ciphertext, and a random IV which was used to encrypt the input. + * + * @param message a UTF-8 encoded message to be encrypted + * @param key A base64 encoded 256 bit symmetric key + * @return A base64 encoded payload containing the IV and AES encrypted ciphertext, which can be decoded by this utility's decrypt method and the same symmetric key + */ + public static String encrypt(String message, String key) throws EncryptionException { + final String ENCRYPT_ALGO = "AES/GCM/NoPadding"; + SecretKey secret = getSecretKeySpec(key); + + return encrypt(message.getBytes(Charset.forName("UTF-8")), + secret, ENCRYPT_ALGO, false); + } + - private static SecretKey getSecretKeySpec(String key) throws EncryptionException { + public static SecretKey getSecretKeySpec(String key) throws EncryptionException { final int KEY_LENGTH_BIT = 256; byte[] keyBytes; try { @@ -69,6 +113,60 @@ private static SecretKey getSecretKeySpec(String key) throws EncryptionException return new SecretKeySpec(keyBytes, "AES"); } + /** + * Decrypts a message and returns the unencrypted byte[] + * The key and transform are specified as inputs + * The IV may be generated to encrypt the input (unless using RSA) + * + * @param bytes a byte[] to be encrypted + * @param key The key that should be used for decryption + * @param transform The transformation to use for decryption + * @param includeMessageLength Whether the message length is included in the packed bytes input + * @return A byte[] containing the unencrypted message + */ + public static byte[] decrypt(byte[] bytes, Key key, String transform, + boolean includeMessageLength) + throws EncryptionException { + final int TAG_LENGTH_BIT = 128; + int readIndex = 0; + int ivLength = bytes[readIndex] & 0xFF; + readIndex++; + byte[] iv = null; + if (ivLength > 0) { + iv = new byte[ivLength]; + System.arraycopy(bytes, readIndex, iv, 0, ivLength); + readIndex += ivLength; + } + + int encryptedLength; + if(includeMessageLength) { + encryptedLength= (bytes[readIndex] & 0xFF) << 8; + readIndex++; + encryptedLength += (bytes[readIndex] & 0xFF); + readIndex++; + } else { + encryptedLength = bytes.length - readIndex; + } + + byte[] encrypted = new byte[encryptedLength]; + System.arraycopy(bytes, readIndex, encrypted, 0, encryptedLength); + + try { + Cipher cipher = Cipher.getInstance(transform); + + if (includeMessageLength) { + cipher.init(Cipher.DECRYPT_MODE, key, iv != null ? new IvParameterSpec(iv) : null); + } else { + cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(TAG_LENGTH_BIT, iv)); + } + + return cipher.doFinal(encrypted); + } catch(NoSuchAlgorithmException | NoSuchPaddingException | IllegalBlockSizeException | + InvalidAlgorithmParameterException | BadPaddingException | InvalidKeyException e) { + throw new EncryptionException("Decrypting message failed", e); + } + } + /** * Decrypts a base64 payload containing an IV and AES encrypted ciphertext using the provided key * @@ -78,27 +176,26 @@ private static SecretKey getSecretKeySpec(String key) throws EncryptionException */ public static String decrypt(String message, String key) throws EncryptionException { final String ENCRYPT_ALGO = "AES/GCM/NoPadding"; - final int TAG_LENGTH_BIT = 128; SecretKey secret = getSecretKeySpec(key); try { byte[] messageBytes = Base64.decode(message); - ByteBuffer bb = ByteBuffer.wrap(messageBytes); - int iv_length_byte = bb.get() & 0xFF; - byte[] iv = new byte[iv_length_byte]; - bb.get(iv); + byte[] plainText = decrypt(messageBytes, secret, ENCRYPT_ALGO, false); - byte[] cipherText = new byte[bb.remaining()]; - bb.get(cipherText); - - - Cipher cipher = Cipher.getInstance(ENCRYPT_ALGO); - cipher.init(Cipher.DECRYPT_MODE, secret, new GCMParameterSpec(TAG_LENGTH_BIT, iv)); - byte[] plainText = cipher.doFinal(cipherText); return new String(plainText, Charset.forName("UTF-8")); - } catch (NoSuchAlgorithmException | BadPaddingException | NoSuchPaddingException | - IllegalBlockSizeException | InvalidAlgorithmParameterException | InvalidKeyException | Base64DecoderException e) { - throw new EncryptionException("Error encountered while decrypting the message", e); + } catch(Base64DecoderException e) { + throw new EncryptionException("Decrypting message failed", e); + } + } + + public static String getMd5HashAsString(String plainText) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(plainText.getBytes()); + byte[] hashInBytes = md.digest(); + return Base64.encode(hashInBytes); + } catch (NoSuchAlgorithmException e) { + return ""; } } diff --git a/src/main/java/org/commcare/util/LogTypes.java b/src/main/java/org/commcare/util/LogTypes.java index 6a5ffdf906..39a1c40d1c 100644 --- a/src/main/java/org/commcare/util/LogTypes.java +++ b/src/main/java/org/commcare/util/LogTypes.java @@ -109,11 +109,16 @@ public class LogTypes { public static final String TYPE_NETWORK = "commcare-network"; - /** - * Logs related to Firebase Cloud Messaging - */ + + /** + * Logs related to Firebase Cloud Messaging + */ public static final String TYPE_FCM = "fcm"; public static final String TYPE_MEDIA_EVENT = "media-event"; + /** + * A Java Exception log + */ + public static final String TYPE_EXCEPTION = "exception"; } diff --git a/src/main/java/org/commcare/xml/DetailFieldParser.java b/src/main/java/org/commcare/xml/DetailFieldParser.java index f0070640d3..b5b4d9ce9c 100644 --- a/src/main/java/org/commcare/xml/DetailFieldParser.java +++ b/src/main/java/org/commcare/xml/DetailFieldParser.java @@ -30,8 +30,11 @@ public DetailFieldParser(KXmlParser parser, GraphParser graphParser, String id) @Override public DetailField parse() throws InvalidStructureException, IOException, XmlPullParserException { checkNode("field"); - DetailField.Builder builder = new DetailField.Builder(); + boolean cacheEnabled = Boolean.parseBoolean(parser.getAttributeValue(null, "cache_enabled")); + builder.setCacheEnabled(cacheEnabled); + boolean lazyLoading = Boolean.parseBoolean(parser.getAttributeValue(null, "lazy_loading")); + builder.setLazyLoading(lazyLoading); String sortDefault = parser.getAttributeValue(null, "sort"); if (sortDefault != null && sortDefault.equals("default")) { diff --git a/src/main/java/org/commcare/xml/DetailParser.java b/src/main/java/org/commcare/xml/DetailParser.java index ca6568e6cd..3706710aaa 100755 --- a/src/main/java/org/commcare/xml/DetailParser.java +++ b/src/main/java/org/commcare/xml/DetailParser.java @@ -39,7 +39,8 @@ public Detail parse() throws InvalidStructureException, IOException, XmlPullPars String forceLandscapeView = parser.getAttributeValue(null, "force-landscape"); String printTemplatePath = parser.getAttributeValue(null, "print-template"); String relevancy = parser.getAttributeValue(null, "relevant"); - boolean isLazyLoading = Boolean.parseBoolean(parser.getAttributeValue(null, "lazy_loading")); + boolean cacheEnabled = Boolean.parseBoolean(parser.getAttributeValue(null, "cache_enabled")); + boolean lazyLoading = Boolean.parseBoolean(parser.getAttributeValue(null, "lazy_loading")); // First fetch the title getNextTagInBlock("detail"); @@ -141,7 +142,7 @@ public Detail parse() throws InvalidStructureException, IOException, XmlPullPars return new Detail(id, title, noItemsText, nodeset, subdetails, fields, variables, actions, callout, fitAcross, useUniformUnits, forceLandscapeView, focusFunction, printTemplatePath, - relevancy, global, detailGroup, isLazyLoading, selectText); + relevancy, global, detailGroup, lazyLoading, cacheEnabled, selectText); } protected DetailParser getDetailParser() { diff --git a/src/main/java/org/javarosa/core/api/ILogger.java b/src/main/java/org/javarosa/core/api/ILogger.java index 073694c7ab..585bb4ddee 100644 --- a/src/main/java/org/javarosa/core/api/ILogger.java +++ b/src/main/java/org/javarosa/core/api/ILogger.java @@ -30,4 +30,6 @@ public interface ILogger { int logSize(); void halt(); + + void logException(Throwable e); } diff --git a/src/main/java/org/javarosa/core/model/instance/DataInstance.java b/src/main/java/org/javarosa/core/model/instance/DataInstance.java index 724f2e7cd7..7b1382d47c 100644 --- a/src/main/java/org/javarosa/core/model/instance/DataInstance.java +++ b/src/main/java/org/javarosa/core/model/instance/DataInstance.java @@ -245,12 +245,16 @@ public void setName(String name) { this.name = name; } + @Override public String toString() { String name = "NULL"; if (this.name != null) { name = this.name; } - return name; + return "DataInstance{" + + "name='" + name + '\'' + + ", instanceid='" + instanceid + '\'' + + '}'; } @Override diff --git a/src/main/java/org/javarosa/core/model/instance/ExternalDataInstance.java b/src/main/java/org/javarosa/core/model/instance/ExternalDataInstance.java index 93a25493fd..4d4a0bd3ac 100644 --- a/src/main/java/org/javarosa/core/model/instance/ExternalDataInstance.java +++ b/src/main/java/org/javarosa/core/model/instance/ExternalDataInstance.java @@ -164,4 +164,13 @@ public void copyFromSource(ExternalDataInstanceSource source) { public ExternalDataInstance copy() { return new ExternalDataInstance(this); } + + @Override + public String toString() { + return "ExternalDataInstance{" + + "reference='" + reference + '\'' + + ", name='" + name + '\'' + + ", instanceid='" + instanceid + '\'' + + '}'; + } } diff --git a/src/main/java/org/javarosa/core/model/utils/GeoPointUtils.java b/src/main/java/org/javarosa/core/model/utils/GeoPointUtils.java index a2bb0ae7e1..a69a718e06 100644 --- a/src/main/java/org/javarosa/core/model/utils/GeoPointUtils.java +++ b/src/main/java/org/javarosa/core/model/utils/GeoPointUtils.java @@ -1,7 +1,11 @@ package org.javarosa.core.model.utils; +import org.gavaghan.geodesy.GlobalCoordinates; import org.javarosa.core.model.data.GeoPointData; +import java.util.ArrayList; +import java.util.List; + /** * Static utility methods for GeoPointData. * @@ -62,4 +66,43 @@ private static double arcHav(double x) { private static double havDistance(double lat1, double lat2, double dLng) { return hav(lat1 - lat2) + hav(dLng) * Math.cos(lat1) * Math.cos(lat2); } + + /** + * Checks if coordinates are within valid bounds for latitude and longitude. + * + * @param latitude Latitude in degrees + * @param longitude Longitude in degrees + * @throws IllegalArgumentException if values are outside geographic bounds + */ + public static void validateCoordinates(double latitude, double longitude) { + if ((latitude < -90.0 || latitude > 90.0) || (longitude < -180.0 || longitude > 180.0)) { + throw new IllegalArgumentException("Invalid coordinates"); + } + } + + /** + * Creates a point list from a flat list of lat/lon strings. + * + * @param latLongList Flat list of lat/lon values (e.g., [lat1, lon1, lat2, lon2, ...]) + * @return List of GlobalCoordinates representing the list of points + * @throws IllegalArgumentException if input is invalid (odd number of elements) + */ + public static List createPointList(List latLongList) throws IllegalArgumentException { + if (latLongList == null || latLongList.size() % 2 != 0) { + throw new IllegalArgumentException( + "Input must contain a list of lat/lng pairs, and must be even-sized."); + } + + int numPoints = latLongList.size() / 2; + List pointList = new ArrayList<>(); + + for (int i = 0; i < numPoints; i++) { + double latitude = Double.parseDouble(latLongList.get(i * 2)); + double longitude = Double.parseDouble(latLongList.get(i * 2 + 1)); + validateCoordinates(latitude, longitude); + pointList.add(new GlobalCoordinates(latitude, longitude)); + } + + return pointList; + } } diff --git a/src/main/java/org/javarosa/core/model/utils/PolygonUtils.java b/src/main/java/org/javarosa/core/model/utils/PolygonUtils.java index 798663fe07..78163d6af1 100644 --- a/src/main/java/org/javarosa/core/model/utils/PolygonUtils.java +++ b/src/main/java/org/javarosa/core/model/utils/PolygonUtils.java @@ -4,7 +4,6 @@ import org.gavaghan.geodesy.GeodeticCurve; import org.gavaghan.geodesy.GlobalCoordinates; -import java.util.ArrayList; import java.util.List; /** @@ -21,23 +20,10 @@ public class PolygonUtils { * @throws IllegalArgumentException if input is invalid or polygon is malformed */ public static List createPolygon(List latLongList) throws IllegalArgumentException { - if (latLongList == null || latLongList.size() < 6 || latLongList.size() % 2 != 0) { - throw new IllegalArgumentException( - "Input must contain at least three lat/lng pairs (six elements total), and must be even-sized."); - } - - int numPoints = latLongList.size() / 2; - List polygon = new ArrayList<>(); - - for (int i = 0; i < numPoints; i++) { - double latitude = Double.parseDouble(latLongList.get(i * 2)); - double longitude = Double.parseDouble(latLongList.get(i * 2 + 1)); - validateCoordinates(latitude, longitude); - polygon.add(new GlobalCoordinates(latitude, longitude)); - } + List polygon = GeoPointUtils.createPointList(latLongList); // Close polygon if not already closed - if (!polygon.get(0).equals(polygon.get(polygon.size() - 1))) { + if (polygon.size() > 2 && !polygon.get(0).equals(polygon.get(polygon.size() - 1))) { polygon.add(new GlobalCoordinates( polygon.get(0).getLatitude(), polygon.get(0).getLongitude())); @@ -50,19 +36,6 @@ public static List createPolygon(List latLongList) th return polygon; } - /** - * Checks if coordinates are within valid bounds for latitude and longitude. - * - * @param latitude Latitude in degrees - * @param longitude Longitude in degrees - * @throws IllegalArgumentException if values are outside geographic bounds - */ - public static void validateCoordinates(double latitude, double longitude) { - if ((latitude < -90.0 || latitude > 90.0) || (longitude < -180.0 || longitude > 180.0)) { - throw new IllegalArgumentException("Invalid polygon coordinates"); - } - } - /** * Computes the closest point on the polygon border to a given test point using geodesic projection. * diff --git a/src/main/java/org/javarosa/core/services/Logger.java b/src/main/java/org/javarosa/core/services/Logger.java index d684778d54..740f01e817 100644 --- a/src/main/java/org/javarosa/core/services/Logger.java +++ b/src/main/java/org/javarosa/core/services/Logger.java @@ -1,5 +1,6 @@ package org.javarosa.core.services; +import org.commcare.util.LogTypes; import org.javarosa.core.api.ILogger; import org.javarosa.core.log.FatalException; import org.javarosa.core.log.WrappedException; @@ -52,7 +53,19 @@ public static void log(String type, String message) { public static void exception(String info, Throwable e) { e.printStackTrace(); - log("exception", (info != null ? info + ": " : "") + WrappedException.printException(e)); + info = info != null ? info + ": " : ""; + log(LogTypes.TYPE_EXCEPTION, info + WrappedException.printException(e)); + if (logger != null) { + try { + String message = e.getMessage(); + if (message == null) { + message = ""; + } + logger.logException(new Exception(info + message, e)); + } catch (RuntimeException ex) { + logger.panic(); + } + } } public static void die(String thread, Exception e) { diff --git a/src/main/java/org/javarosa/core/services/storage/IStorageUtilityIndexed.java b/src/main/java/org/javarosa/core/services/storage/IStorageUtilityIndexed.java index 9fc30ba7cb..a4d865b920 100644 --- a/src/main/java/org/javarosa/core/services/storage/IStorageUtilityIndexed.java +++ b/src/main/java/org/javarosa/core/services/storage/IStorageUtilityIndexed.java @@ -279,6 +279,16 @@ void bulkRead(LinkedHashSet cuedCases, HashMap recordMap) */ Vector getBulkRecordsForIndex(String metaFieldName, Collection matchingValues); + + /** + * Bulk retrieves a set of the record ids in storage based on a list of values matching one of the + * field for this storage + * @param metaFieldName field we are matching against + * @param matchingValues matching values for metaFieldName that we want to filter records against + * @return A Vector of Externalizable objects e, such that the field specified is equal to the corresponding value provided. + */ + Vector getBulkIdsForIndex(String metaFieldName, Collection matchingValues); + /** * Provide public accessor to the inner class that is stored */ diff --git a/src/main/java/org/javarosa/core/services/storage/util/DummyIndexedStorageUtility.java b/src/main/java/org/javarosa/core/services/storage/util/DummyIndexedStorageUtility.java index 33874b4d9f..c319abc856 100644 --- a/src/main/java/org/javarosa/core/services/storage/util/DummyIndexedStorageUtility.java +++ b/src/main/java/org/javarosa/core/services/storage/util/DummyIndexedStorageUtility.java @@ -367,6 +367,13 @@ public Vector getBulkRecordsForIndex(String metaFieldName, Collection return getRecordsForValues(new String[]{metaFieldName}, matchingValues.toArray()); } + @Override + public Vector getBulkIdsForIndex(String metaFieldName, Collection matchingValues) { + // we don't care about bulk retrieval for dummy storage, so just call normal method to get records here + List result = getIDsForValues(new String[]{metaFieldName}, matchingValues.toArray()); + return new Vector<>(result); + } + @Override public void bulkReadMetadata(LinkedHashSet cuedCases, String[] metaDataIds, HashMap metadataMap) { for (int i : ((LinkedHashSet)cuedCases)) { diff --git a/src/main/java/org/javarosa/core/util/DAG.java b/src/main/java/org/javarosa/core/util/DAG.java index 1752cfdf3a..b6c75b4312 100644 --- a/src/main/java/org/javarosa/core/util/DAG.java +++ b/src/main/java/org/javarosa/core/util/DAG.java @@ -3,6 +3,8 @@ import java.util.Enumeration; import java.util.HashSet; import java.util.Hashtable; +import java.util.LinkedList; +import java.util.Set; import java.util.Stack; import java.util.Vector; @@ -138,6 +140,40 @@ public Stack getSinks() { return roots; } + /** + * Find all nodes reachable from the given set of source nodes + * + * @param sourceNodes The set of starting nodes + * @return A set containing all reachable nodes + */ + public Set findConnectedRecords(Set sourceNodes) { + Set visited = new HashSet<>(); + LinkedList queue = new LinkedList<>(sourceNodes); + while (!queue.isEmpty()) { + I current = queue.poll(); + if(visited.contains(current)){ + continue; + } + visited.add(current); + enqueueUnvisitedNeighbors(edges, current, queue, visited); + enqueueUnvisitedNeighbors(inverseEdges, current, queue, visited); + } + return visited; + } + + // Adds unvisited neighboring nodes of the given record to the queue for further traversal + private void enqueueUnvisitedNeighbors(Hashtable>> edges, I current, LinkedList queue, + Set visited) { + if (edges.containsKey(current)) { + Vector> neighbors = edges.get(current); + for (Edge neighbor : neighbors) { + if(!visited.contains(neighbor.i)){ + queue.add(neighbor.i); + } + } + } + } + public static class Edge { public final I i; public final E e; diff --git a/src/main/java/org/javarosa/model/xform/DataModelSerializer.java b/src/main/java/org/javarosa/model/xform/DataModelSerializer.java index 3efc6f59ec..387640ac5f 100644 --- a/src/main/java/org/javarosa/model/xform/DataModelSerializer.java +++ b/src/main/java/org/javarosa/model/xform/DataModelSerializer.java @@ -61,7 +61,7 @@ public void serialize(AbstractTreeElement root) throws IOException { serializer.flush(); } - private void serializeNode(AbstractTreeElement instanceNode) throws IOException { + public void serializeNode(AbstractTreeElement instanceNode) throws IOException { //don't serialize template nodes or non-relevant nodes if (!instanceNode.isRelevant() || instanceNode.getMult() == TreeReference.INDEX_TEMPLATE) { return; @@ -88,4 +88,4 @@ private void serializeAttributes(AbstractTreeElement instanceNode) throws IOExce serializer.attribute(instanceNode.getAttributeNamespace(i), instanceNode.getAttributeName(i), val); } } -} \ No newline at end of file +} diff --git a/src/main/java/org/javarosa/xpath/expr/XPathClosestPointOnPolygonFunc.java b/src/main/java/org/javarosa/xpath/expr/XPathClosestPointOnPolygonFunc.java index 9a56bd9a4e..a497c3614d 100644 --- a/src/main/java/org/javarosa/xpath/expr/XPathClosestPointOnPolygonFunc.java +++ b/src/main/java/org/javarosa/xpath/expr/XPathClosestPointOnPolygonFunc.java @@ -5,6 +5,7 @@ import org.javarosa.core.model.data.GeoPointData; import org.javarosa.core.model.data.UncastData; import org.javarosa.core.model.instance.DataInstance; +import org.javarosa.core.model.utils.GeoPointUtils; import org.javarosa.core.model.utils.PolygonUtils; import org.javarosa.xpath.XPathException; import org.javarosa.xpath.XPathTypeMismatchException; @@ -24,9 +25,9 @@ * *

Parameters:

*
    - *
  • polygon_coords: A space-separated string of lon/lat pairs (e.g. "'78.041309 27.174957 78 - * .042574 27.174884 78.042661 27.175493 78.041383 27.175569'")
    • - *
  • point_coord: A single point as "lon lat eg('78.043 27.175)"
    • + *
  • point_coord: A single point as "lat lon eg('27.175 78.043')"
    • + *
  • polygon_coords: A space-separated string of lat/lon pairs (e.g. "'27.174957 78.041309 27 + * .174884 78.042574 27.175493 78.042661 27.175569 78.041383'")
    • *
* *

Returns:

@@ -35,9 +36,9 @@ * *

Recommended Use:

* 
- *     closest-point-on-polygon('78.041 27.176','78.041309 27.174957 78.042574 27.174884 78.042661 27.175493 78.041383 27.175569')
+ *     closest-point-on-polygon('27.176 78.041','27.174957 78.041309 27.174884 78.042574 27.175493 78.042661 27.175569 78.041383')
  *
- *

This example finds the closest point on the polygon to (78.041, 27.176)

+ *

This example finds the closest point on the polygon to (27.176 78.041)

*/ public class XPathClosestPointOnPolygonFunc extends XPathFuncExpr { public static final String NAME = "closest-point-on-polygon"; @@ -74,7 +75,7 @@ private static String closestPointToPolygon(Object from, Object to) { String[] coordinates = inputPolygon.split(" "); List polygon = PolygonUtils.createPolygon(Arrays.asList(coordinates)); GeoPointData pointData = new GeoPointData().cast(new UncastData(inputPoint)); - PolygonUtils.validateCoordinates(pointData.getLatitude(), pointData.getLongitude()); + GeoPointUtils.validateCoordinates(pointData.getLatitude(), pointData.getLongitude()); GlobalCoordinates pointCoordinates = new GlobalCoordinates(pointData.getLatitude(), pointData.getLongitude()); return PolygonUtils.findClosestPoint(pointCoordinates, polygon).toString(); @@ -86,4 +87,4 @@ private static String closestPointToPolygon(Object from, Object to) { throw new XPathException(e.getMessage()); } } -} \ No newline at end of file +} diff --git a/src/main/java/org/javarosa/xpath/expr/XPathExpression.java b/src/main/java/org/javarosa/xpath/expr/XPathExpression.java index 86e2b27ffe..c395360e19 100755 --- a/src/main/java/org/javarosa/xpath/expr/XPathExpression.java +++ b/src/main/java/org/javarosa/xpath/expr/XPathExpression.java @@ -91,8 +91,9 @@ private static void serializeElements(XPathNodeset nodeset, OutputStream output) for (TreeReference ref : refs) { AbstractTreeElement treeElement = instance.resolveReference(ref); - s.serialize(treeElement); + s.serializeNode(treeElement); } + serializer.flush(); } /** diff --git a/src/main/java/org/javarosa/xpath/expr/XPathIsPointInsidePolygonFunc.java b/src/main/java/org/javarosa/xpath/expr/XPathIsPointInsidePolygonFunc.java index d9c39073dc..6de573bee1 100644 --- a/src/main/java/org/javarosa/xpath/expr/XPathIsPointInsidePolygonFunc.java +++ b/src/main/java/org/javarosa/xpath/expr/XPathIsPointInsidePolygonFunc.java @@ -5,6 +5,7 @@ import org.javarosa.core.model.data.GeoPointData; import org.javarosa.core.model.data.UncastData; import org.javarosa.core.model.instance.DataInstance; +import org.javarosa.core.model.utils.GeoPointUtils; import org.javarosa.core.model.utils.PolygonUtils; import org.javarosa.xpath.XPathException; import org.javarosa.xpath.XPathTypeMismatchException; @@ -69,7 +70,7 @@ private static boolean isPointWithinBoundary(Object from, Object to) { String[] coordinates = inputPolygon.split(" "); List polygon = PolygonUtils.createPolygon(Arrays.asList(coordinates)); GeoPointData pointData = new GeoPointData().cast(new UncastData(inputPoint)); - PolygonUtils.validateCoordinates(pointData.getLatitude(), pointData.getLongitude()); + GeoPointUtils.validateCoordinates(pointData.getLatitude(), pointData.getLongitude()); GlobalCoordinates pointCoordinates = new GlobalCoordinates(pointData.getLatitude(), pointData.getLongitude()); return PolygonUtils.isPointInsideOrOnPolygon(pointCoordinates, polygon); diff --git a/src/test/java/org/commcare/backend/suite/model/test/AppStructureTests.java b/src/test/java/org/commcare/backend/suite/model/test/AppStructureTests.java index 1374dcb069..8ac8009d6c 100644 --- a/src/test/java/org/commcare/backend/suite/model/test/AppStructureTests.java +++ b/src/test/java/org/commcare/backend/suite/model/test/AppStructureTests.java @@ -256,9 +256,18 @@ public void testDetailWithShading() { } @Test - public void testDetailWithLazyLoadingSet() { + public void testDetailPerformanceAttributes() { Detail detail = mApp.getSession().getPlatform().getDetail("m0_case_short"); assertTrue(detail.isLazyLoading()); + assertTrue(detail.isCacheEnabled()); + assertTrue(detail.getFields()[0].isCacheEnabled()); + assertFalse(detail.getFields()[0].isLazyLoading()); + assertTrue(detail.getFields()[1].isLazyLoading()); + assertFalse(detail.getFields()[1].isCacheEnabled()); + + Detail detailNoCaching = mApp.getSession().getPlatform().getDetail("m1_case_short"); + assertFalse(detailNoCaching.isCacheEnabled()); + assertFalse(detailNoCaching.isCacheEnabled()); } @Test diff --git a/src/test/java/org/commcare/test/utilities/CasePurgeTest.java b/src/test/java/org/commcare/test/utilities/CasePurgeTest.java index 3c90e76b31..5a6e5717e5 100644 --- a/src/test/java/org/commcare/test/utilities/CasePurgeTest.java +++ b/src/test/java/org/commcare/test/utilities/CasePurgeTest.java @@ -1,11 +1,14 @@ package org.commcare.test.utilities; +import static org.commcare.cases.util.CasePurgeFilter.getFullCaseGraph; + import org.commcare.cases.model.Case; import org.commcare.cases.model.CaseIndex; import org.commcare.cases.util.CasePurgeFilter; import org.commcare.cases.util.InvalidCaseGraphException; import org.javarosa.core.services.storage.IStorageIterator; import org.javarosa.core.services.storage.util.DummyIndexedStorageUtility; +import org.javarosa.core.util.DAG; import org.javarosa.core.util.externalizable.LivePrototypeFactory; import org.json.JSONArray; import org.json.JSONException; @@ -17,8 +20,10 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.Vector; /** @@ -52,11 +57,11 @@ public static Iterable testData() { } private static Object[] parseParametersFromJSONObject(JSONObject root) { - Object[] parameters = new Object[7]; + Object[] parameters = new Object[8]; parameters[0] = root.getString("name"); String[] jsonArrayKeys = - new String[]{"cases", "owned", "closed", "subcases", "extensions", "outcome"}; + new String[]{"cases", "owned", "closed", "subcases", "extensions", "outcome", "relation_outcome"}; for (int i = 0; i < jsonArrayKeys.length; i++) { addJSONArrayIfPresent(root, i+1, jsonArrayKeys[i], parameters); } @@ -76,17 +81,19 @@ private static void addJSONArrayIfPresent(JSONObject root, int index, String key private final HashSet ownedCases = new HashSet<>(); private final HashSet closedCases = new HashSet<>(); private final HashSet outcomeSet = new HashSet<>(); + private final HashMap> relationOutcomeSet = new HashMap<>(); private final ArrayList indices = new ArrayList<>(); public CasePurgeTest(String name, JSONArray cases, JSONArray owned, JSONArray closed, - JSONArray subcases, JSONArray extensions, JSONArray outcome) { + JSONArray subcases, JSONArray extensions, JSONArray outcome, JSONArray relationOutcomes) { this.name = name; - createTestObjectsFromParameters(cases, owned, closed, subcases, extensions, outcome); + createTestObjectsFromParameters(cases, owned, closed, subcases, extensions, outcome, relationOutcomes); } private void createTestObjectsFromParameters(JSONArray casesJson, JSONArray ownedJson, - JSONArray closedJson, JSONArray subcasesJson, - JSONArray extensionsJson, JSONArray outcomeJson) { + JSONArray closedJson, JSONArray subcasesJson, + JSONArray extensionsJson, JSONArray outcomeJson, + JSONArray relationOutcomes) { if (casesJson != null) { getCases(casesJson, cases); } @@ -104,6 +111,23 @@ private void createTestObjectsFromParameters(JSONArray casesJson, JSONArray owne getIndices(extensionsJson, indices, CaseIndex.RELATIONSHIP_EXTENSION); } getCases(outcomeJson, outcomeSet); + if (relationOutcomes != null) { + populateRelationOutcomes(relationOutcomes, outcomeSet); + } + } + + private void populateRelationOutcomes(JSONArray relationOutcomes, HashSet outcomeSet) + throws JSONException { + int count = 0; + for (String outcome : outcomeSet) { + JSONObject relationOutcome = (JSONObject)relationOutcomes.get(count++); + JSONArray relatedCases = relationOutcome.optJSONArray("related_cases"); + HashSet relatedCasesSet = new HashSet<>(); + for (int i = 0; i < relatedCases.length(); ++i) { + relatedCasesSet.add(relatedCases.getString(i)); + } + relationOutcomeSet.put(outcome, relatedCasesSet); + } } private void getIndices(JSONArray indices, ArrayList indexSet, @@ -138,16 +162,26 @@ public void executeTest() throws InvalidCaseGraphException { Vector ownerIds = new Vector<>(); ownerIds.add(userId); - storage.removeAll(new CasePurgeFilter(storage, ownerIds)); + + storage.removeAll(new CasePurgeFilter(getFullCaseGraph(storage, ownerIds))); HashSet inStorage = new HashSet<>(); + // redo the graph as we don't want the eliminated cases anymore + DAG graph = getFullCaseGraph(storage, ownerIds); for (IStorageIterator iterator = storage.iterate(); iterator.hasMore(); ) { Case c = iterator.nextRecord(); - inStorage.add(c.getCaseId()); + String caseId = c.getCaseId(); + inStorage.add(caseId); + + + HashSet relatedCasesSet = relationOutcomeSet.get(caseId); + HashSet input = new HashSet<>(); + input.add(caseId); + Set relatedCases = graph.findConnectedRecords(input); + Assert.assertEquals(name, relatedCasesSet, relatedCases); } Assert.assertEquals(name, outcomeSet, inStorage); - } private void initCaseStorage(DummyIndexedStorageUtility storage, diff --git a/src/test/resources/app_structure/app_strings.txt b/src/test/resources/app_structure/app_strings.txt index 3920c3135c..d4a24c4fc4 100644 --- a/src/test/resources/app_structure/app_strings.txt +++ b/src/test/resources/app_structure/app_strings.txt @@ -1,3 +1,3 @@ m0_no_items_text=Empty List m0_select_text=Continue With Case -m0.case_short.case_starred_1.alt_text.k0=gold star \ No newline at end of file +m0.case_short.case_starred_1.alt_text.k0=gold star diff --git a/src/test/resources/app_structure/suite.xml b/src/test/resources/app_structure/suite.xml index 72c3a538d8..0a35f7eb86 100644 --- a/src/test/resources/app_structure/suite.xml +++ b/src/test/resources/app_structure/suite.xml @@ -11,7 +11,7 @@ - + <text>Case List</text> @@ -66,7 +66,7 @@ - + @@ -94,6 +94,21 @@ + +
+ Name +
+ + + + + + + diff --git a/src/test/resources/case_relationship_tests.json b/src/test/resources/case_relationship_tests.json index 37d84aab01..2806bf73b3 100644 --- a/src/test/resources/case_relationship_tests.json +++ b/src/test/resources/case_relationship_tests.json @@ -22,6 +22,23 @@ "a", "b", "c" + ], + "relation_outcome": [ + { + "related_cases": [ + "a" + ] + }, + { + "related_cases": [ + "b" + ] + }, + { + "related_cases": [ + "c" + ] + } ] }, { @@ -38,6 +55,20 @@ "outcome": [ "d", "b" + ], + "relation_outcome": [ + { + "related_cases": [ + "d", + "b" + ] + }, + { + "related_cases": [ + "b", + "d" + ] + } ] }, { @@ -65,6 +96,20 @@ "outcome": [ "a", "b" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b" + ] + }, + { + "related_cases": [ + "b", + "a" + ] + } ] }, { @@ -84,6 +129,13 @@ ], "outcome": [ "a" + ], + "relation_outcome": [ + { + "related_cases": [ + "a" + ] + } ] }, { @@ -127,6 +179,20 @@ "outcome": [ "d", "b" + ], + "relation_outcome": [ + { + "related_cases": [ + "d", + "b" + ] + }, + { + "related_cases": [ + "b", + "d" + ] + } ] }, { @@ -152,6 +218,13 @@ ], "outcome": [ "b" + ], + "relation_outcome": [ + { + "related_cases": [ + "b" + ] + } ] }, { @@ -168,6 +241,20 @@ "outcome": [ "a", "e" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "e" + ] + }, + { + "related_cases": [ + "e", + "a" + ] + } ] }, { @@ -211,6 +298,29 @@ "a", "b", "e" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b", + "e" + ] + }, + { + "related_cases": [ + "b", + "e", + "a" + ] + }, + { + "related_cases": [ + "e", + "a", + "b" + ] + } ] }, { @@ -263,6 +373,29 @@ "a", "b", "e" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b", + "e" + ] + }, + { + "related_cases": [ + "b", + "e", + "a" + ] + }, + { + "related_cases": [ + "e", + "a", + "b" + ] + } ] }, { @@ -288,6 +421,13 @@ ], "outcome": [ "b" + ], + "relation_outcome": [ + { + "related_cases": [ + "b" + ] + } ] }, { @@ -304,6 +444,20 @@ "outcome": [ "d", "a" + ], + "relation_outcome": [ + { + "related_cases": [ + "d", + "a" + ] + }, + { + "related_cases": [ + "a", + "d" + ] + } ] }, { @@ -320,6 +474,20 @@ "outcome": [ "d", "a" + ], + "relation_outcome": [ + { + "related_cases": [ + "d", + "a" + ] + }, + { + "related_cases": [ + "a", + "d" + ] + } ] }, { @@ -341,6 +509,29 @@ "d", "a", "b" + ], + "relation_outcome": [ + { + "related_cases": [ + "d", + "a", + "b" + ] + }, + { + "related_cases": [ + "a", + "d", + "b" + ] + }, + { + "related_cases": [ + "a", + "d", + "b" + ] + } ] }, { @@ -367,6 +558,40 @@ "b", "c", "d" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b", + "c", + "d" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d" + ] + } ] }, { @@ -400,6 +625,53 @@ "c", "d", "e" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b", + "c", + "d", + "e" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d", + "e" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d", + "e" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d", + "e" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d", + "e" + ] + } ] }, { @@ -418,6 +690,13 @@ ], "outcome": [ "a" + ], + "relation_outcome": [ + { + "related_cases": [ + "a" + ] + } ] }, { @@ -444,6 +723,29 @@ "a", "b", "c" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b", + "c" + ] + }, + { + "related_cases": [ + "a", + "b", + "c" + ] + }, + { + "related_cases": [ + "a", + "b", + "c" + ] + } ] }, { @@ -471,6 +773,29 @@ "a", "b", "c" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b", + "c" + ] + }, + { + "related_cases": [ + "a", + "b", + "c" + ] + }, + { + "related_cases": [ + "a", + "b", + "c" + ] + } ] }, { @@ -497,6 +822,20 @@ "outcome": [ "a", "b" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b" + ] + }, + { + "related_cases": [ + "a", + "b" + ] + } ] }, { @@ -521,6 +860,29 @@ "a", "b", "e" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b", + "e" + ] + }, + { + "related_cases": [ + "a", + "b", + "e" + ] + }, + { + "related_cases": [ + "a", + "b", + "e" + ] + } ] }, { @@ -655,6 +1017,13 @@ ], "outcome": [ "a" + ], + "relation_outcome": [ + { + "related_cases": [ + "a" + ] + } ] }, { @@ -687,6 +1056,53 @@ "c", "d", "e" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b", + "c", + "d", + "e" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d", + "e" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d", + "e" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d", + "e" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d", + "e" + ] + } ] }, { @@ -721,6 +1137,13 @@ ], "outcome": [ "a" + ], + "relation_outcome": [ + { + "related_cases": [ + "a" + ] + } ] }, { @@ -756,6 +1179,29 @@ ], "outcome": [ "a", "b", "c" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b", + "c" + ] + }, + { + "related_cases": [ + "a", + "b", + "c" + ] + }, + { + "related_cases": [ + "a", + "b", + "c" + ] + } ] }, { @@ -773,6 +1219,13 @@ ], "outcome": [ "a" + ], + "relation_outcome": [ + { + "related_cases": [ + "a" + ] + } ] }, { @@ -791,6 +1244,13 @@ ], "outcome": [ "a" + ], + "relation_outcome": [ + { + "related_cases": [ + "a" + ] + } ] }, { @@ -808,6 +1268,40 @@ ], "outcome": [ "d", "a", "c", "e" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "c", + "d", + "e" + ] + }, + { + "related_cases": [ + "a", + "c", + "d", + "e" + ] + }, + { + "related_cases": [ + "a", + "c", + "d", + "e" + ] + }, + { + "related_cases": [ + "a", + "c", + "d", + "e" + ] + } ] }, { @@ -828,6 +1322,40 @@ ], "outcome": [ "a", "b", "c", "d" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b", + "c", + "d" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d" + ] + }, + { + "related_cases": [ + "a", + "b", + "c", + "d" + ] + } ] }, { @@ -848,6 +1376,20 @@ ], "outcome": [ "a", "b" + ], + "relation_outcome": [ + { + "related_cases": [ + "a", + "b" + ] + }, + { + "related_cases": [ + "a", + "b" + ] + } ] }, { @@ -871,6 +1413,38 @@ ], "outcome": [ "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ], + "relation_outcome": [ + { + "related_cases": [ + "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ] + }, + { + "related_cases": [ + "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ] + }, + { + "related_cases": [ + "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ] + }, + { + "related_cases": [ + "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ] + }, + { + "related_cases": [ + "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ] + }, + { + "related_cases": [ + "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ] + } ] }, { @@ -894,6 +1468,38 @@ ], "outcome": [ "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ], + "relation_outcome": [ + { + "related_cases": [ + "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ] + }, + { + "related_cases": [ + "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ] + }, + { + "related_cases": [ + "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ] + }, + { + "related_cases": [ + "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ] + }, + { + "related_cases": [ + "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ] + }, + { + "related_cases": [ + "claim", "house", "p_mother", "p_child", "child_health", "measurement" + ] + } ] }, { @@ -918,6 +1524,18 @@ ], "outcome": [ "claim", "house" + ], + "relation_outcome": [ + { + "related_cases": [ + "claim", "house" + ] + }, + { + "related_cases": [ + "claim", "house" + ] + } ] }, { @@ -942,6 +1560,18 @@ ], "outcome": [ "claim", "house" + ], + "relation_outcome": [ + { + "related_cases": [ + "claim", "house" + ] + }, + { + "related_cases": [ + "claim", "house" + ] + } ] }, { @@ -963,6 +1593,33 @@ ], "outcome": [ "L", "B", "C", "D", "E" + ], + "relation_outcome": [ + { + "related_cases": [ + "L", "B", "C", "D", "E" + ] + }, + { + "related_cases": [ + "L", "B", "C", "D", "E" + ] + }, + { + "related_cases": [ + "L", "B", "C", "D", "E" + ] + }, + { + "related_cases": [ + "L", "B", "C", "D", "E" + ] + }, + { + "related_cases": [ + "L", "B", "C", "D", "E" + ] + } ] }, { @@ -978,6 +1635,13 @@ ], "outcome": [ "parent" + ], + "relation_outcome": [ + { + "related_cases": [ + "parent" + ] + } ] }