From f1bac0b9fc3ea950f9280e56a9c0f93eaed66cee Mon Sep 17 00:00:00 2001 From: Christian Melchior Date: Tue, 31 Mar 2020 22:58:13 +0200 Subject: [PATCH 1/7] Upgrade to latest core --- dependencies.list | 4 ++-- realm/realm-library/src/main/cpp/object-store | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dependencies.list b/dependencies.list index 3fb3f9d7bf..7a86581a12 100644 --- a/dependencies.list +++ b/dependencies.list @@ -1,7 +1,7 @@ # Realm Sync release used by Realm Java (This includes Realm Core) # https://github.com/realm/realm-sync/releases -REALM_SYNC_VERSION=10.0.0-alpha.4 -REALM_SYNC_SHA256=7048eff89f00554aa4014239a25d419fc98d0b22074ff2deadd3e9717a5c4dee +REALM_SYNC_VERSION=10.0.0-alpha.5 +REALM_SYNC_SHA256=aa490300e06dca385622bc83430f9d8e3141fcccfd7bf6ae2f68af8a29af5dc1 # Object Server Release used by Integration tests. Installed using NPM. # Use `npm view realm-object-server versions` to get a list of available versions. diff --git a/realm/realm-library/src/main/cpp/object-store b/realm/realm-library/src/main/cpp/object-store index 2b94e623c5..f18b240b8b 160000 --- a/realm/realm-library/src/main/cpp/object-store +++ b/realm/realm-library/src/main/cpp/object-store @@ -1 +1 @@ -Subproject commit 2b94e623c506d0d6fc96a8fd087429fc715830d3 +Subproject commit f18b240b8bf592e5d5ae8045c5444f423b6d8e9d From fd95cc417dd5d6e4fbfca092e4b5528a4feba46d Mon Sep 17 00:00:00 2001 From: Christian Melchior Date: Tue, 31 Mar 2020 23:06:44 +0200 Subject: [PATCH 2/7] Fix compiler errors --- .../cpp/io_realm_internal_objectstore_OsSyncUser.cpp | 12 ++++++------ .../src/objectServer/java/io/realm/RealmUser.java | 4 ++-- .../io/realm/internal/objectstore/OsSyncUser.java | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/realm/realm-library/src/main/cpp/io_realm_internal_objectstore_OsSyncUser.cpp b/realm/realm-library/src/main/cpp/io_realm_internal_objectstore_OsSyncUser.cpp index 15278b1dc5..8afed94bea 100644 --- a/realm/realm-library/src/main/cpp/io_realm_internal_objectstore_OsSyncUser.cpp +++ b/realm/realm-library/src/main/cpp/io_realm_internal_objectstore_OsSyncUser.cpp @@ -188,8 +188,8 @@ JNIEXPORT jbyte JNICALL Java_io_realm_internal_objectstore_OsSyncUser_nativeGetS auto user = *reinterpret_cast*>(j_native_ptr); switch(user->state()) { case SyncUser::State::LoggedOut: return static_cast(io_realm_internal_objectstore_OsSyncUser_STATE_LOGGED_OUT); - case SyncUser::State::Active: return static_cast(io_realm_internal_objectstore_OsSyncUser_STATE_ACTIVE); - case SyncUser::State::Error: return static_cast(io_realm_internal_objectstore_OsSyncUser_STATE_ERROR); + case SyncUser::State::LoggedIn: return static_cast(io_realm_internal_objectstore_OsSyncUser_STATE_LOGGED_IN); + case SyncUser::State::Removed: return static_cast(io_realm_internal_objectstore_OsSyncUser_STATE_REMOVED); default: throw std::logic_error(util::format("Unknown state: %1", static_cast(user->state()))); } @@ -206,11 +206,11 @@ JNIEXPORT void JNICALL Java_io_realm_internal_objectstore_OsSyncUser_nativeSetSt case io_realm_internal_objectstore_OsSyncUser_STATE_LOGGED_OUT: user->set_state(SyncUser::State::LoggedOut); break; - case io_realm_internal_objectstore_OsSyncUser_STATE_ACTIVE: - user->set_state(SyncUser::State::Active); + case io_realm_internal_objectstore_OsSyncUser_STATE_LOGGED_IN: + user->set_state(SyncUser::State::LoggedIn); break; - case io_realm_internal_objectstore_OsSyncUser_STATE_ERROR: - user->set_state(SyncUser::State::Error); + case io_realm_internal_objectstore_OsSyncUser_STATE_REMOVED: + user->set_state(SyncUser::State::Removed); break; default: throw std::logic_error(util::format("Unknown state: %1", j_state)); diff --git a/realm/realm-library/src/objectServer/java/io/realm/RealmUser.java b/realm/realm-library/src/objectServer/java/io/realm/RealmUser.java index 883010e23c..98218315ae 100644 --- a/realm/realm-library/src/objectServer/java/io/realm/RealmUser.java +++ b/realm/realm-library/src/objectServer/java/io/realm/RealmUser.java @@ -51,8 +51,8 @@ public String getKey() { } public enum State { - ACTIVE(OsSyncUser.STATE_ACTIVE), - ERROR(OsSyncUser.STATE_ERROR), + ACTIVE(OsSyncUser.STATE_LOGGED_IN), + ERROR(OsSyncUser.STATE_REMOVED), LOGGED_OUT(OsSyncUser.STATE_LOGGED_OUT); private final byte nativeValue; diff --git a/realm/realm-library/src/objectServer/java/io/realm/internal/objectstore/OsSyncUser.java b/realm/realm-library/src/objectServer/java/io/realm/internal/objectstore/OsSyncUser.java index 4f1a3aa9cb..662b366091 100644 --- a/realm/realm-library/src/objectServer/java/io/realm/internal/objectstore/OsSyncUser.java +++ b/realm/realm-library/src/objectServer/java/io/realm/internal/objectstore/OsSyncUser.java @@ -23,8 +23,8 @@ public class OsSyncUser implements NativeObject { private final long nativePtr; private static final long nativeFinalizerPtr = nativeGetFinalizerMethodPtr(); - public static final byte STATE_ACTIVE = 1; - public static final byte STATE_ERROR = 2; + public static final byte STATE_LOGGED_IN = 1; + public static final byte STATE_REMOVED = 2; public static final byte STATE_LOGGED_OUT = 3; public OsSyncUser(long nativePtr) { @@ -107,7 +107,7 @@ public byte getState() { } public void invalidate() { - nativeSetState(nativePtr, STATE_ERROR); + nativeSetState(nativePtr, STATE_REMOVED); } public String getProviderType() { From e2504786f71f269256fd4f7e38e2120acf06b1eb Mon Sep 17 00:00:00 2001 From: Christian Melchior Date: Wed, 1 Apr 2020 21:45:01 +0200 Subject: [PATCH 3/7] Add functionality for backing up Realms in case of test failures --- Jenkinsfile | 11 ++++ .../java/io/realm/RealmInMemoryTest.java | 12 +++- .../androidTest/java/io/realm/RealmTests.java | 1 - .../testUtils/java/io/realm/TestHelper.java | 56 +++++++++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index acc93e005b..dfa138bfea 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -151,6 +151,7 @@ try { } } finally { archiveServerLogs(mongoDbRealmContainer.id, mongoDbRealmCommandServerContainer.id) + archiveBackupFiles() mongoDbRealmContainer.stop() mongoDbRealmCommandServerContainer.stop() sh "docker network rm ${dockerNetworkId}" @@ -233,6 +234,16 @@ def archiveServerLogs(String mongoDbRealmContainerId, String commandServerContai sh 'rm mongodb.log' } +def archiveBackupFiles() { + sh 'adb pull /data/data/io.realm.test/files/realm-backup || true' + zip([ + 'zipFile': 'realm-backups.zip', + 'archive': true, + 'dir' : 'realm-backup' + ]) + sh 'rm -r realm-backup' +} + def sendMetrics(String metricName, String metricValue, Map tags) { def tagsString = getTagsString(tags) withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: '5b8ad2d9-61a4-43b5-b4df-b8ff6b1f16fa', passwordVariable: 'influx_pass', usernameVariable: 'influx_user']]) { diff --git a/realm/realm-library/src/androidTest/java/io/realm/RealmInMemoryTest.java b/realm/realm-library/src/androidTest/java/io/realm/RealmInMemoryTest.java index 8ca8360eae..f6da6ad33b 100644 --- a/realm/realm-library/src/androidTest/java/io/realm/RealmInMemoryTest.java +++ b/realm/realm-library/src/androidTest/java/io/realm/RealmInMemoryTest.java @@ -40,7 +40,6 @@ import static junit.framework.Assert.assertTrue; import static junit.framework.Assert.fail; -@Ignore("FIXME: See https://github.com/realm/realm-java/issues/6790") @RunWith(AndroidJUnit4.class) public class RealmInMemoryTest { @@ -167,8 +166,15 @@ public void writeCopyTo() { onDiskRealm.close(); // Tests a encrypted Realm file. - testRealm.writeEncryptedCopyTo(new File(configFactory.getRoot(), encFileName), key); - onDiskRealm = Realm.getInstance(encConf); + testRealm.writeEncryptedCopyTo(new File(testDir, encFileName), key); + try { + onDiskRealm = Realm.getInstance(encConf); + } catch (RealmFileException ex) { + // Attempt to backup the file in case of a crash. + // See https://github.com/realm/realm-java/issues/6790 + TestHelper.backupRealm(encConf); + throw ex; + } assertEquals(1, onDiskRealm.where(Dog.class).count()); onDiskRealm.close(); // Tests with a wrong key to see if it fails as expected. diff --git a/realm/realm-library/src/androidTest/java/io/realm/RealmTests.java b/realm/realm-library/src/androidTest/java/io/realm/RealmTests.java index 016dffcc00..fad546a606 100644 --- a/realm/realm-library/src/androidTest/java/io/realm/RealmTests.java +++ b/realm/realm-library/src/androidTest/java/io/realm/RealmTests.java @@ -4562,7 +4562,6 @@ public void hittingMaxNumberOfVersionsThrows() { } // Test for https://github.com/realm/realm-java/issues/6152 - @Ignore("FIXME: https://github.com/realm/realm-java/issues/6792") @Test @RunTestInLooperThread public void encryption_stressTest() { diff --git a/realm/realm-library/src/testUtils/java/io/realm/TestHelper.java b/realm/realm-library/src/testUtils/java/io/realm/TestHelper.java index 2468ef74e8..984de54be5 100644 --- a/realm/realm-library/src/testUtils/java/io/realm/TestHelper.java +++ b/realm/realm-library/src/testUtils/java/io/realm/TestHelper.java @@ -20,6 +20,8 @@ import android.content.res.AssetManager; import android.os.Build; import android.os.Looper; +import android.util.Base64; + import androidx.test.platform.app.InstrumentationRegistry; import org.junit.Assert; @@ -34,6 +36,9 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.nio.charset.Charset; +import java.nio.file.CopyOption; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Date; @@ -64,6 +69,7 @@ import io.realm.internal.Util; import io.realm.internal.async.RealmThreadPoolExecutor; import io.realm.log.LogLevel; +import io.realm.log.RealmLog; import io.realm.log.RealmLogger; import io.realm.rule.TestRealmConfigurationFactory; @@ -378,6 +384,56 @@ public static String getRandomString(int length) { return sb.toString(); } + // Attempts to backup a Realm file, so it can be retrieve after any tests finished + public static void backupRealm(RealmConfiguration config) { + try { + Path source = new File(config.getPath()).toPath(); + File backupDir = new File(BaseRealm.applicationContext.getFilesDir(), "realm-backup"); + if (!backupDir.exists() && !backupDir.mkdir()) { + throw new IllegalStateException("Could not create backup folder"); + } + File dest = new File(backupDir, config.getRealmFileName()); + int i = 0; + while (dest.exists()) { + dest = new File(backupDir, config.getRealmFileName() + "_" + i); + } + Files.copy(source, dest.toPath()); + if (config.getEncryptionKey() != null) { + RealmLog.error("Successfully backup of encrypted Realm: %s to %s. The encryption key is: \"%s\"", + source.toAbsolutePath(), dest.getAbsolutePath(), bytesToHex(config.getEncryptionKey())); + } else { + RealmLog.error("Successfully backup up: %s to %s", source.toAbsolutePath(), dest.getAbsolutePath()); + } + } catch(Exception ex) { + RealmLog.error("Attempted to backup file, but it failed.", ex); + } + } + + //Original source: https://stackoverflow.com/a/9855338/1389357 + private final static char[] hexArray = "0123456789ABCDEF".toCharArray(); + public static String bytesToHex(byte[] bytes) { + char[] hexChars = new char[bytes.length * 2]; + for ( int j = 0; j < bytes.length; j++ ) { + int v = bytes[j] & 0xFF; + hexChars[j * 2] = hexArray[v >>> 4]; + hexChars[j * 2 + 1] = hexArray[v & 0x0F]; + } + return new String(hexChars); + } + + public static byte[] hexToBytes(String data) { + char[] hex = data.toCharArray(); + byte[] raw = new byte[hex.length / 2]; + for (int src = 0, dst = 0; dst < raw.length; ++dst) { + int hi = Character.digit(hex[src++], 16); + int lo = Character.digit(hex[src++], 16); + if ((hi < 0) || (lo < 0)) + throw new IllegalArgumentException(); + raw[dst] = (byte) (hi << 4 | lo); + } + return raw; + } + /** * Returns a naive logger that can be used to test the values that are sent to the logger. */ From 7cd0d367f8a35ed3da42788905e541557e55fb69 Mon Sep 17 00:00:00 2001 From: Christian Melchior Date: Wed, 1 Apr 2020 22:29:57 +0200 Subject: [PATCH 4/7] Fix Jenkins issues --- Jenkinsfile | 16 +++++++++------- .../java/io/realm/RealmInMemoryTest.java | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dfa138bfea..a9ccf22dc8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -235,13 +235,15 @@ def archiveServerLogs(String mongoDbRealmContainerId, String commandServerContai } def archiveBackupFiles() { - sh 'adb pull /data/data/io.realm.test/files/realm-backup || true' - zip([ - 'zipFile': 'realm-backups.zip', - 'archive': true, - 'dir' : 'realm-backup' - ]) - sh 'rm -r realm-backup' + sh 'adb pull /data/data/io.realm.test/files/realm-backup' + if (fileExists('realm-backup')) { + zip([ + 'zipFile': 'realm-backups.zip', + 'archive': true, + 'dir' : 'realm-backup' + ]) + sh 'rm -r realm-backup' + } } def sendMetrics(String metricName, String metricValue, Map tags) { diff --git a/realm/realm-library/src/androidTest/java/io/realm/RealmInMemoryTest.java b/realm/realm-library/src/androidTest/java/io/realm/RealmInMemoryTest.java index f6da6ad33b..e1371adbf2 100644 --- a/realm/realm-library/src/androidTest/java/io/realm/RealmInMemoryTest.java +++ b/realm/realm-library/src/androidTest/java/io/realm/RealmInMemoryTest.java @@ -166,7 +166,7 @@ public void writeCopyTo() { onDiskRealm.close(); // Tests a encrypted Realm file. - testRealm.writeEncryptedCopyTo(new File(testDir, encFileName), key); + testRealm.writeEncryptedCopyTo(new File(configFactory.getRoot(), encFileName), key); try { onDiskRealm = Realm.getInstance(encConf); } catch (RealmFileException ex) { From a826e569aa54ee14f83e0d9dce448222140c2ca1 Mon Sep 17 00:00:00 2001 From: Christian Melchior Date: Thu, 2 Apr 2020 12:40:04 +0200 Subject: [PATCH 5/7] toPath not available below Android 26 --- .../testUtils/java/io/realm/TestHelper.java | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/realm/realm-library/src/testUtils/java/io/realm/TestHelper.java b/realm/realm-library/src/testUtils/java/io/realm/TestHelper.java index 984de54be5..46856f2bc5 100644 --- a/realm/realm-library/src/testUtils/java/io/realm/TestHelper.java +++ b/realm/realm-library/src/testUtils/java/io/realm/TestHelper.java @@ -29,9 +29,12 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; @@ -387,28 +390,29 @@ public static String getRandomString(int length) { // Attempts to backup a Realm file, so it can be retrieve after any tests finished public static void backupRealm(RealmConfiguration config) { try { - Path source = new File(config.getPath()).toPath(); + File source = new File(config.getPath()); File backupDir = new File(BaseRealm.applicationContext.getFilesDir(), "realm-backup"); if (!backupDir.exists() && !backupDir.mkdir()) { throw new IllegalStateException("Could not create backup folder"); } - File dest = new File(backupDir, config.getRealmFileName()); + File destination = new File(backupDir, config.getRealmFileName()); int i = 0; - while (dest.exists()) { - dest = new File(backupDir, config.getRealmFileName() + "_" + i); + while (destination.exists()) { + destination = new File(backupDir, config.getRealmFileName() + "_" + i); } - Files.copy(source, dest.toPath()); + copy(source, destination); if (config.getEncryptionKey() != null) { RealmLog.error("Successfully backup of encrypted Realm: %s to %s. The encryption key is: \"%s\"", - source.toAbsolutePath(), dest.getAbsolutePath(), bytesToHex(config.getEncryptionKey())); + source.getAbsolutePath(), destination.getAbsolutePath(), bytesToHex(config.getEncryptionKey())); } else { - RealmLog.error("Successfully backup up: %s to %s", source.toAbsolutePath(), dest.getAbsolutePath()); + RealmLog.error("Successfully backup up: %s to %s", source.getAbsolutePath(), destination.getAbsolutePath()); } } catch(Exception ex) { RealmLog.error("Attempted to backup file, but it failed.", ex); } } + //Original source: https://stackoverflow.com/a/9855338/1389357 private final static char[] hexArray = "0123456789ABCDEF".toCharArray(); public static String bytesToHex(byte[] bytes) { @@ -434,6 +438,26 @@ public static byte[] hexToBytes(String data) { return raw; } + // Credit: https://stackoverflow.com/questions/9292954/how-to-make-a-copy-of-a-file-in-android + public static void copy(File src, File dst) throws IOException { + InputStream in = new FileInputStream(src); + try { + OutputStream out = new FileOutputStream(dst); + try { + // Transfer bytes from in to out + byte[] buf = new byte[1024]; + int len; + while ((len = in.read(buf)) > 0) { + out.write(buf, 0, len); + } + } finally { + out.close(); + } + } finally { + in.close(); + } + } + /** * Returns a naive logger that can be used to test the values that are sent to the logger. */ From 95b30378b466f42f14220a932e087501501ace7f Mon Sep 17 00:00:00 2001 From: Christian Melchior Date: Thu, 2 Apr 2020 17:15:57 +0200 Subject: [PATCH 6/7] Trigger CI --- realm/realm-library/src/testUtils/java/io/realm/TestHelper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/realm/realm-library/src/testUtils/java/io/realm/TestHelper.java b/realm/realm-library/src/testUtils/java/io/realm/TestHelper.java index 46856f2bc5..5b59cae614 100644 --- a/realm/realm-library/src/testUtils/java/io/realm/TestHelper.java +++ b/realm/realm-library/src/testUtils/java/io/realm/TestHelper.java @@ -412,7 +412,6 @@ public static void backupRealm(RealmConfiguration config) { } } - //Original source: https://stackoverflow.com/a/9855338/1389357 private final static char[] hexArray = "0123456789ABCDEF".toCharArray(); public static String bytesToHex(byte[] bytes) { From f3d31f087caebeed3daed9cf957983b29cc226e7 Mon Sep 17 00:00:00 2001 From: Christian Melchior Date: Fri, 3 Apr 2020 09:21:15 +0200 Subject: [PATCH 7/7] Backup files where ADB is available --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a9ccf22dc8..3d35c9e790 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -119,6 +119,7 @@ try { stopLogCatCollector(backgroundPid) storeJunitResults 'realm/realm-library/build/outputs/androidTest-results/connected/**/TEST-*.xml' storeJunitResults 'realm/kotlin-extensions/build/outputs/androidTest-results/connected/**/TEST-*.xml' + archiveRealmBackupFiles() } } @@ -151,7 +152,6 @@ try { } } finally { archiveServerLogs(mongoDbRealmContainer.id, mongoDbRealmCommandServerContainer.id) - archiveBackupFiles() mongoDbRealmContainer.stop() mongoDbRealmCommandServerContainer.stop() sh "docker network rm ${dockerNetworkId}" @@ -234,7 +234,7 @@ def archiveServerLogs(String mongoDbRealmContainerId, String commandServerContai sh 'rm mongodb.log' } -def archiveBackupFiles() { +def archiveRealmBackupFiles() { sh 'adb pull /data/data/io.realm.test/files/realm-backup' if (fileExists('realm-backup')) { zip([