From bd7495ba6dcf35e86f71a5000032b250d83553a2 Mon Sep 17 00:00:00 2001
From: mic074b <mic704b@gmail.com>
Date: Tue, 10 Mar 2020 21:12:32 +1100
Subject: [PATCH] Test harness fixes.

---
 CliClient/app/cli-integration-tests.js        |   2 +-
 CliClient/tests/integration_ShowAllNotes.js   |   2 +
 CliClient/tests/test-utils.js                 |  53 +++-
 ReactNativeClient/lib/BaseApplication.js      |   6 +-
 ReactNativeClient/lib/BaseModel.js            |   4 +
 ReactNativeClient/lib/registry.js             | 242 ++++++++++--------
 .../lib/services/SearchEngine.js              |   1 +
 ReactNativeClient/root.js                     |   2 +-
 8 files changed, 183 insertions(+), 129 deletions(-)

diff --git a/CliClient/app/cli-integration-tests.js b/CliClient/app/cli-integration-tests.js
index 426118ad..5059ee24 100644
--- a/CliClient/app/cli-integration-tests.js
+++ b/CliClient/app/cli-integration-tests.js
@@ -218,7 +218,7 @@ async function main() {
 	logger.info(await execCommand(client, 'version'));
 
 	await db.open({ name: `${client.profileDir}/database.sqlite` });
-	BaseModel.db_ = db;
+	BaseModel.setDb(db);
 	await Setting.load();
 
 	let onlyThisTest = 'testMv';
diff --git a/CliClient/tests/integration_ShowAllNotes.js b/CliClient/tests/integration_ShowAllNotes.js
index 0c5d22b4..c5f7ad34 100644
--- a/CliClient/tests/integration_ShowAllNotes.js
+++ b/CliClient/tests/integration_ShowAllNotes.js
@@ -50,6 +50,7 @@ describe('integration_ShowAllNotes', function() {
 		let notes0 = await createNTestNotes(3, folders[0]);
 		let notes1 = await createNTestNotes(3, folders[1]);
 		let notes2 = await createNTestNotes(3, folders[2]);
+		await time.msleep(100);
 
 		// TEST ACTION: View all-notes
 		testApp.dispatch({ type: 'SMART_FILTER_SELECT', id: ALL_NOTES_FILTER_ID });
@@ -67,6 +68,7 @@ describe('integration_ShowAllNotes', function() {
 		let folders = await createNTestFolders(2);
 		let notes0 = await createNTestNotes(3, folders[0]);
 		let notes1 = await createNTestNotes(3, folders[1]);
+		await time.msleep(100);
 		testApp.dispatch({ type: 'FOLDER_SELECT', id: id(folders[1]) });
 		await time.msleep(100);
 		testApp.dispatch({ type: 'NOTE_SELECT',	id: id(notes1[1]) });
diff --git a/CliClient/tests/test-utils.js b/CliClient/tests/test-utils.js
index d26aedb2..56ce4996 100644
--- a/CliClient/tests/test-utils.js
+++ b/CliClient/tests/test-utils.js
@@ -125,6 +125,10 @@ function sleep(n) {
 	});
 }
 
+function currentClientId() {
+	return currentClient_;
+}
+
 async function switchClient(id) {
 	if (!databases_[id]) throw new Error(`Call setupDatabaseAndSynchronizer(${id}) first!!`);
 
@@ -132,12 +136,7 @@ async function switchClient(id) {
 	await Setting.saveAll();
 
 	currentClient_ = id;
-	BaseModel.db_ = databases_[id];
-	Folder.db_ = databases_[id];
-	Note.db_ = databases_[id];
-	BaseItem.db_ = databases_[id];
-	Setting.db_ = databases_[id];
-	Resource.db_ = databases_[id];
+	BaseModel.setDb(databases_[id]);
 
 	BaseItem.encryptionService_ = encryptionServices_[id];
 	Resource.encryptionService_ = encryptionServices_[id];
@@ -189,6 +188,7 @@ async function setupDatabase(id = null) {
 	Setting.cache_ = null;
 
 	if (databases_[id]) {
+		BaseModel.setDb(databases_[id]);
 		await clearDatabase(id);
 		await Setting.load();
 		if (!Setting.value('clientId')) Setting.setValue('clientId', uuid.create());
@@ -207,7 +207,7 @@ async function setupDatabase(id = null) {
 	databases_[id].setLogger(dbLogger);
 	await databases_[id].open({ name: filePath });
 
-	BaseModel.db_ = databases_[id];
+	BaseModel.setDb(databases_[id]);
 	await Setting.load();
 	if (!Setting.value('clientId')) Setting.setValue('clientId', uuid.create());
 }
@@ -438,6 +438,7 @@ async function createNTestFolders(n) {
 	for (let i = 0; i < n; i++) {
 		let folder = await Folder.save({ title: 'folder' });
 		folders.push(folder);
+		await time.msleep(100);
 	}
 	return folders;
 }
@@ -448,10 +449,12 @@ async function createNTestNotes(n, folder, tagIds = null, title = 'note') {
 		let title_ = n > 1 ? `${title}${i}` : title;
 		let note = await Note.save({ title: title_, parent_id: folder.id, is_conflict: 0 });
 		notes.push(note);
+		await time.msleep(100);
 	}
 	if (tagIds) {
 		for (let i = 0; i < notes.length; i++) {
 			await Tag.setNoteTagsByIds(notes[i].id, tagIds);
+			await time.msleep(100);
 		}
 	}
 	return notes;
@@ -462,25 +465,41 @@ async function createNTestTags(n) {
 	for (let i = 0; i < n; i++) {
 		let tag = await Tag.save({ title: 'tag' });
 		tags.push(tag);
+		await time.msleep(100);
 	}
 	return tags;
 }
 
-// Integration test application
+// Application for feature integration testing
 class TestApp extends BaseApplication {
-	constructor() {
+	constructor(hasGui = true) {
 		super();
+		this.hasGui_ = hasGui;
 		this.middlewareCalls_ = [];
+		this.logger_ = super.logger();
+	}
+
+	hasGui() {
+		return this.hasGui_;
 	}
 
 	async start(argv) {
+		this.logger_.info('Test app starting...');
 		await clearDatabase(); // not sure why we need this as we use our own database
 
-		argv = argv.concat(['--profile', `tests-build/profile-${uuid.create()}`]);
+		if (!argv.includes('--profile')) {
+			argv = argv.concat(['--profile', `tests-build/profile/${uuid.create()}`]);
+		}
 		argv = await super.start(['',''].concat(argv));
+		// For now, disable sync and encryption to avoid spurious intermittent failures
+		// caused by them interupting processing and causing delays.
+		Setting.setValue('sync.interval', 0);
+		Setting.setValue('encryption.enabled', false);
 		this.initRedux();
 		Setting.dispatchUpdateAll();
-		await time.msleep(100);
+		ItemChange.waitForAllSaved();
+		this.logger_.info('Test app started...');
+		await time.msleep(250);
 	}
 
 	async generalMiddleware(store, next, action) {
@@ -503,13 +522,19 @@ class TestApp extends BaseApplication {
 		});
 	}
 
+	async profileDir() {
+		return await Setting.value('profileDir');
+	}
+
 	async destroy() {
+		this.logger_.info('Test app destroying...');
 		await this.waitForMiddleware_();
 		this.deinitRedux();
+		ItemChange.waitForAllSaved();
+		this.logger_.info('Test app destroyed...');
 		await super.destroy();
+		await time.msleep(100);
 	}
 }
 
-
-module.exports = { kvStore, resourceService, allSyncTargetItemsEncrypted, setupDatabase, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, asyncTest, id, ids, sortedIds, at, createNTestNotes, createNTestFolders, createNTestTags, TestApp };
-
+module.exports = { kvStore, resourceService, allSyncTargetItemsEncrypted, setupDatabase, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, asyncTest, currentClientId, id, ids, sortedIds, at, createNTestNotes, createNTestFolders, createNTestTags, TestApp };
diff --git a/ReactNativeClient/lib/BaseApplication.js b/ReactNativeClient/lib/BaseApplication.js
index 6dcc9a2f..e2e5f923 100644
--- a/ReactNativeClient/lib/BaseApplication.js
+++ b/ReactNativeClient/lib/BaseApplication.js
@@ -64,9 +64,11 @@ class BaseApplication {
 		await DecryptionWorker.instance().destroy();
 		await FoldersScreenUtils.cancelTimers();
 		await reg.cancelTimers();
+		EncryptionService.instance_ = null;
 
 		this.eventEmitter_.removeAllListeners();
-		BaseModel.db_ = null;
+		KvStore.instance_ = null;
+		BaseModel.setDb(null);
 		reg.setDb(null);
 
 		this.logger_ = null;
@@ -645,7 +647,7 @@ class BaseApplication {
 		// if (Setting.value('env') === 'dev') await this.database_.clearForTesting();
 
 		reg.setDb(this.database_);
-		BaseModel.db_ = this.database_;
+		BaseModel.setDb(this.database_);
 
 		await Setting.load();
 
diff --git a/ReactNativeClient/lib/BaseModel.js b/ReactNativeClient/lib/BaseModel.js
index bb507aaf..1f755b3f 100644
--- a/ReactNativeClient/lib/BaseModel.js
+++ b/ReactNativeClient/lib/BaseModel.js
@@ -12,6 +12,10 @@ class BaseModel {
 		throw new Error('Must be overriden');
 	}
 
+	static setDb(db) {
+		this.db_ = db;
+	}
+
 	static addModelMd(model) {
 		if (!model) return model;
 
diff --git a/ReactNativeClient/lib/registry.js b/ReactNativeClient/lib/registry.js
index 63488987..83a08706 100644
--- a/ReactNativeClient/lib/registry.js
+++ b/ReactNativeClient/lib/registry.js
@@ -52,135 +52,142 @@ reg.syncTarget = (syncTargetId = null) => {
 	return target;
 };
 
-reg.scheduleSync_ = async (delay = null, syncOptions = null) => {
-	if (delay === null) delay = 1000 * 10;
-	if (syncOptions === null) syncOptions = {};
-
-	let promiseResolve = null;
-	const promise = new Promise((resolve) => {
-		promiseResolve = resolve;
-	});
-
-	if (reg.scheduleSyncId_) {
-		clearTimeout(reg.scheduleSyncId_);
-		reg.scheduleSyncId_ = null;
-	}
+reg.scheduleSync = async (delay = null, syncOptions = null) => {
+	reg.schedSyncCalls_.push(true);
 
-	reg.logger().info('Scheduling sync operation...');
+	try {
+		if (delay === null) delay = 1000 * 10;
+		if (syncOptions === null) syncOptions = {};
 
-	if (Setting.value('env') === 'dev' && delay !== 0) {
-		reg.logger().info('Schedule sync DISABLED!!!');
-		return;
-	}
+		let promiseResolve = null;
+		const promise = new Promise((resolve) => {
+			promiseResolve = resolve;
+		});
 
-	const timeoutCallback = async () => {
-		reg.scheduleSyncId_ = null;
-		reg.logger().info('Preparing scheduled sync');
+		if (reg.scheduleSyncId_) {
+			clearTimeout(reg.scheduleSyncId_);
+			reg.scheduleSyncId_ = null;
+		}
 
-		const syncTargetId = Setting.value('sync.target');
+		reg.logger().info('Scheduling sync operation...', delay);
 
-		if (!(await reg.syncTarget(syncTargetId).isAuthenticated())) {
-			reg.logger().info('Synchroniser is missing credentials - manual sync required to authenticate.');
-			promiseResolve();
+		if (Setting.value('env') === 'dev' && delay !== 0) {
+			reg.logger().info('Schedule sync DISABLED!!!');
 			return;
 		}
 
-		try {
-			const sync = await reg.syncTarget(syncTargetId).synchronizer();
-
-			const contextKey = `sync.${syncTargetId}.context`;
-			let context = Setting.value(contextKey);
+		const timeoutCallback = async () => {
+			reg.timerCallbackCalls_.push(true);
 			try {
-				context = context ? JSON.parse(context) : {};
-			} catch (error) {
-				// Clearing the context is inefficient since it means all items are going to be re-downloaded
-				// however it won't result in duplicate items since the synchroniser is going to compare each
-				// item to the current state.
-				reg.logger().warn(`Could not parse JSON sync context ${contextKey}:`, context);
-				reg.logger().info('Clearing context and starting from scratch');
-				context = null;
-			}
+				reg.scheduleSyncId_ = null;
+				reg.logger().info('Preparing scheduled sync');
 
-			try {
-				reg.logger().info('Starting scheduled sync');
-				const options = Object.assign({}, syncOptions, { context: context });
-				if (!options.saveContextHandler) {
-					options.saveContextHandler = newContext => {
-						Setting.setValue(contextKey, JSON.stringify(newContext));
-					};
-				}
-				const newContext = await sync.start(options);
-				Setting.setValue(contextKey, JSON.stringify(newContext));
-			} catch (error) {
-				if (error.code == 'alreadyStarted') {
-					reg.logger().info(error.message);
-				} else {
+				const syncTargetId = Setting.value('sync.target');
+
+				if (!(await reg.syncTarget(syncTargetId).isAuthenticated())) {
+					reg.logger().info('Synchroniser is missing credentials - manual sync required to authenticate.');
 					promiseResolve();
-					throw error;
+					return;
 				}
-			}
-		} catch (error) {
-			reg.logger().info('Could not run background sync:');
-			reg.logger().info(error);
-
-			// Special case to display OneDrive Business error. This is the full error that's received when trying to use a OneDrive Business account:
-			//
-			// {"error":"invalid_client","error_description":"AADSTS50011: The reply address 'http://localhost:1917' does not match the reply addresses configured for
-			// the application: 'cbabb902-d276-4ea4-aa88-062a5889d6dc'. More details: not specified\r\nTrace ID: 6e63dac6-8b37-47e2-bd1b-4768f8713400\r\nCorrelation
-			// ID: acfd6503-8d97-4349-ae2e-e7a19dd7b6bc\r\nTimestamp: 2017-12-01 13:35:55Z","error_codes":[50011],"timestamp":"2017-12-01 13:35:55Z","trace_id":
-			// "6e63dac6-8b37-47e2-bd1b-4768f8713400","correlation_id":"acfd6503-8d97-4349-ae2e-e7a19dd7b6bc"}: TOKEN: null Error: {"error":"invalid_client",
-			// "error_description":"AADSTS50011: The reply address 'http://localhost:1917' does not match the reply addresses configured for the application:
-			// 'cbabb902-d276-4ea4-aa88-062a5889d6dc'. More details: not specified\r\nTrace ID: 6e63dac6-8b37-47e2-bd1b-4768f8713400\r\nCorrelation ID
-			//  acfd6503-8d97-4349-ae2e-e7a19dd7b6bc\r\nTimestamp: 2017-12-01 13:35:55Z","error_codes":[50011],"timestamp":"2017-12-01 13:35:55Z","trace_id":
-			// "6e63dac6-8b37-47e2-bd1b-4768f8713400","correlation_id":"acfd6503-8d97-4349-ae2e-e7a19dd7b6bc"}
-			if (error && error.message && error.message.indexOf('"invalid_client"') >= 0) {
-				reg.showErrorMessageBox(_('Could not synchronize with OneDrive.\n\nThis error often happens when using OneDrive for Business, which unfortunately cannot be supported.\n\nPlease consider using a regular OneDrive account.'));
-			}
-		}
-
-		reg.setupRecurrentSync();
 
-		promiseResolve();
-	};
+				try {
+					const sync = await reg.syncTarget(syncTargetId).synchronizer();
+
+					const contextKey = `sync.${syncTargetId}.context`;
+					let context = Setting.value(contextKey);
+					try {
+						context = context ? JSON.parse(context) : {};
+					} catch (error) {
+						// Clearing the context is inefficient since it means all items are going to be re-downloaded
+						// however it won't result in duplicate items since the synchroniser is going to compare each
+						// item to the current state.
+						reg.logger().warn(`Could not parse JSON sync context ${contextKey}:`, context);
+						reg.logger().info('Clearing context and starting from scratch');
+						context = null;
+					}
+
+					try {
+						reg.logger().info('Starting scheduled sync');
+						const options = Object.assign({}, syncOptions, { context: context });
+						if (!options.saveContextHandler) {
+							options.saveContextHandler = newContext => {
+								Setting.setValue(contextKey, JSON.stringify(newContext));
+							};
+						}
+						const newContext = await sync.start(options);
+						Setting.setValue(contextKey, JSON.stringify(newContext));
+					} catch (error) {
+						if (error.code == 'alreadyStarted') {
+							reg.logger().info(error.message);
+						} else {
+							promiseResolve();
+							throw error;
+						}
+					}
+				} catch (error) {
+					reg.logger().info('Could not run background sync:');
+					reg.logger().info(error);
+
+					// Special case to display OneDrive Business error. This is the full error that's received when trying to use a OneDrive Business account:
+					//
+					// {"error":"invalid_client","error_description":"AADSTS50011: The reply address 'http://localhost:1917' does not match the reply addresses configured for
+					// the application: 'cbabb902-d276-4ea4-aa88-062a5889d6dc'. More details: not specified\r\nTrace ID: 6e63dac6-8b37-47e2-bd1b-4768f8713400\r\nCorrelation
+					// ID: acfd6503-8d97-4349-ae2e-e7a19dd7b6bc\r\nTimestamp: 2017-12-01 13:35:55Z","error_codes":[50011],"timestamp":"2017-12-01 13:35:55Z","trace_id":
+					// "6e63dac6-8b37-47e2-bd1b-4768f8713400","correlation_id":"acfd6503-8d97-4349-ae2e-e7a19dd7b6bc"}: TOKEN: null Error: {"error":"invalid_client",
+					// "error_description":"AADSTS50011: The reply address 'http://localhost:1917' does not match the reply addresses configured for the application:
+					// 'cbabb902-d276-4ea4-aa88-062a5889d6dc'. More details: not specified\r\nTrace ID: 6e63dac6-8b37-47e2-bd1b-4768f8713400\r\nCorrelation ID
+					//  acfd6503-8d97-4349-ae2e-e7a19dd7b6bc\r\nTimestamp: 2017-12-01 13:35:55Z","error_codes":[50011],"timestamp":"2017-12-01 13:35:55Z","trace_id":
+					// "6e63dac6-8b37-47e2-bd1b-4768f8713400","correlation_id":"acfd6503-8d97-4349-ae2e-e7a19dd7b6bc"}
+					if (error && error.message && error.message.indexOf('"invalid_client"') >= 0) {
+						reg.showErrorMessageBox(_('Could not synchronize with OneDrive.\n\nThis error often happens when using OneDrive for Business, which unfortunately cannot be supported.\n\nPlease consider using a regular OneDrive account.'));
+					}
+				}
+				reg.setupRecurrentSync();
+				promiseResolve();
 
-	if (delay === 0) {
-		timeoutCallback();
-	} else {
-		reg.scheduleSyncId_ = setTimeout(timeoutCallback, delay);
-	}
+			} finally {
+				reg.timerCallbackCalls_.pop();
+			}
+		};
 
-	return promise;
-};
+		if (delay === 0) {
+			timeoutCallback();
+		} else {
+			reg.scheduleSyncId_ = setTimeout(timeoutCallback, delay);
+		}
+		return promise;
 
-reg.scheduleSync = async (delay = null, syncOptions = null) => {
-	reg.syncCalls_.push(true);
-	try {
-		await reg.scheduleSync_(delay, syncOptions);
 	} finally {
-		reg.syncCalls_.pop();
+		reg.schedSyncCalls_.pop();
 	}
 };
 
 reg.setupRecurrentSync = () => {
-	if (reg.recurrentSyncId_) {
-		shim.clearInterval(reg.recurrentSyncId_);
-		reg.recurrentSyncId_ = null;
-	}
+	reg.setupRecurrentCalls_.push(true);
 
-	if (!Setting.value('sync.interval')) {
-		reg.logger().debug('Recurrent sync is disabled');
-	} else {
-		reg.logger().debug(`Setting up recurrent sync with interval ${Setting.value('sync.interval')}`);
-
-		if (Setting.value('env') === 'dev') {
-			reg.logger().info('Recurrent sync operation DISABLED!!!');
-			return;
+	try {
+		if (reg.recurrentSyncId_) {
+			shim.clearInterval(reg.recurrentSyncId_);
+			reg.recurrentSyncId_ = null;
 		}
 
-		reg.recurrentSyncId_ = shim.setInterval(() => {
-			reg.logger().info('Running background sync on timer...');
-			reg.scheduleSync(0);
-		}, 1000 * Setting.value('sync.interval'));
+		if (!Setting.value('sync.interval')) {
+			reg.logger().debug('Recurrent sync is disabled');
+		} else {
+			reg.logger().debug(`Setting up recurrent sync with interval ${Setting.value('sync.interval')}`);
+
+			if (Setting.value('env') === 'dev') {
+				reg.logger().info('Recurrent sync operation DISABLED!!!');
+				return;
+			}
+
+			reg.recurrentSyncId_ = shim.setInterval(() => {
+				reg.logger().info('Running background sync on timer...');
+				reg.scheduleSync(0);
+			}, 1000 * Setting.value('sync.interval'));
+		}
+	} finally {
+		reg.setupRecurrentCalls_.pop();
 	}
 };
 
@@ -192,21 +199,34 @@ reg.db = () => {
 	return reg.db_;
 };
 
-reg.cancelTimers = async () => {
+reg.cancelTimers_ = () => {
 	if (this.recurrentSyncId_) {
-		clearTimeout(this.recurrentSyncId_);
+		shim.clearInterval(reg.recurrentSyncId_);
 		this.recurrentSyncId_ = null;
 	}
+	if (reg.scheduleSyncId_) {
+		clearTimeout(reg.scheduleSyncId_);
+		reg.scheduleSyncId_ = null;
+	}
+};
+
+reg.cancelTimers = async () => {
+	reg.logger().info('Cancelling sync timers');
+	reg.cancelTimers_();
+
 	return new Promise((resolve) => {
-		const iid = setInterval(() => {
-			if (!reg.syncCalls_.length) {
-				clearInterval(iid);
+		setInterval(() => {
+			// ensure processing complete
+			if (!reg.setupRecurrentCalls_.length && !reg.schedSyncCalls_.length && !reg.timerCallbackCalls_.length) {
+				reg.cancelTimers_();
 				resolve();
 			}
 		}, 100);
 	});
 };
 
-reg.syncCalls_ = [];
+reg.setupRecurrentCalls_ = [];
+reg.schedSyncCalls_ = [];
+reg.timerCallbackCalls_ = [];
 
 module.exports = { reg };
diff --git a/ReactNativeClient/lib/services/SearchEngine.js b/ReactNativeClient/lib/services/SearchEngine.js
index 9b71f8b7..f2c0e80a 100644
--- a/ReactNativeClient/lib/services/SearchEngine.js
+++ b/ReactNativeClient/lib/services/SearchEngine.js
@@ -424,6 +424,7 @@ class SearchEngine {
 			const iid = setInterval(() => {
 				if (!this.syncCalls_.length) {
 					clearInterval(iid);
+					this.instance_ = null;
 					resolve();
 				}
 			}, 100);
diff --git a/ReactNativeClient/root.js b/ReactNativeClient/root.js
index d848cb76..21b5e916 100644
--- a/ReactNativeClient/root.js
+++ b/ReactNativeClient/root.js
@@ -412,7 +412,7 @@ async function initialize(dispatch) {
 	FoldersScreenUtils.dispatch = dispatch;
 	BaseSyncTarget.dispatch = dispatch;
 	NavService.dispatch = dispatch;
-	BaseModel.db_ = db;
+	BaseModel.setDb(db);
 
 	KvStore.instance().setDb(reg.db());
 
-- 
2.17.1

