reorganize into separate crates

- taskchampion -- core implementation of a replica
 - taskchampion-cli -- command-line interface
 - taskchampion-sync-server -- server implementation (not much yet!)

taskchampion/tests/data/tdb2-test.data

@@ -0,0 +1,2 @@
+[description:"https:\/\/phabricator.services.example.com\/D7364 &open;taskgraph&close; Download debian packages" end:"1541705209" entry:"1538520624" modified:"1541705209" phabricatorid:"D7364" priority:"M" project:"moz" status:"completed" tags:"phabricator,respond" uuid:"ca33f6d6-1688-4503-90be-3b3526a32b5a" wait:"1570118809"]
+[annotation_1541461824:"https:\/\/github.com\/taskcluster\/taskcluster-worker-manager\/pull\/3" description:"https:\/\/github.com\/taskcluster\/taskcluster-worker-manager\/pull\/3 More changes" end:"1541702602" entry:"1541451283" githubbody:"some notes:\n\n1.  This is a huge PR, so I'm not expecting a quick turn around at all.  If you have questions, let me know and I can hope on Vidyo.\n1. Data persistence is written in a semi-janky way.  My intention is to use the time while this is under review to make progress on postgres stuff so that the next review cycle will be using new Postgres things. Which means....  There's a lot of bugs in the concurrency because there's no synchronisation at all in this janky-ish model.\n1. The API is the minimum api required to get provisioning-ish things working\n1. I intend to write a system for automatically testing provider and bidding strategy implementations, so that you can do instantiate a provider\/strategy, stub\/spy it as needed then run a test suite against it and have it do its thing.  The idea is that each provider will need to mock their underlying api system in their own way, but the set of tests we run for Provider API conformance would be pretty standardized.  This should make writing tests for new providers a lot easier.\n1. The provider\/strategy loading system is intentionally simple.  The idea is that these aren't general purpose plugins, but rather special ones.  The idea is that the config files would essentially declare instances and then provide constructor arguments to initialize them all... This would make enabling\/disabling providers\/strategies fairly trivial\n1. I decided to drop fake implementations of providers and strategies for testing the provisioning logic and instead opt for Sinon stubs, which I think give us a better testing story\n1. I still intend to have fake providers and bidding strategies for doing API testing.\n\nLet me know, and again, I don't expect or need a quick turn around on this PR.\n" githubcreatedon:"1541451283" githubnamespace:"djmitche" githubnumber:"3.000000" githubrepo:"taskcluster\/taskcluster-worker-manager" githubtitle:"More changes" githubtype:"pull_request" githubupdatedat:"1541699191" githuburl:"https:\/\/github.com\/taskcluster\/taskcluster-worker-manager\/pull\/3" githubuser:"jhford" modified:"1541702602" priority:"H" project:"moz" status:"completed" tags:"respond" uuid:"2186f981-d1f5-4642-b833-5b16b3a2d334"]

taskchampion/tests/operation_transform_invariant.rs

@@ -0,0 +1,85 @@
+use chrono::Utc;
+use proptest::prelude::*;
+use taskchampion::{taskstorage, Operation, DB};
+use uuid::Uuid;
+
+fn newdb() -> DB {
+    DB::new(Box::new(taskstorage::InMemoryStorage::new()))
+}
+
+fn uuid_strategy() -> impl Strategy<Value = Uuid> {
+    prop_oneof![
+        Just(Uuid::parse_str("83a2f9ef-f455-4195-b92e-a54c161eebfc").unwrap()),
+        Just(Uuid::parse_str("56e0be07-c61f-494c-a54c-bdcfdd52d2a7").unwrap()),
+        Just(Uuid::parse_str("4b7ed904-f7b0-4293-8a10-ad452422c7b3").unwrap()),
+        Just(Uuid::parse_str("9bdd0546-07c8-4e1f-a9bc-9d6299f4773b").unwrap()),
+    ]
+}
+
+fn operation_strategy() -> impl Strategy<Value = Operation> {
+    prop_oneof![
+        uuid_strategy().prop_map(|uuid| Operation::Create { uuid }),
+        uuid_strategy().prop_map(|uuid| Operation::Delete { uuid }),
+        (uuid_strategy(), "(title|project|status)").prop_map(|(uuid, property)| {
+            Operation::Update {
+                uuid,
+                property,
+                value: Some("true".into()),
+                timestamp: Utc::now(),
+            }
+        }),
+    ]
+}
+
+proptest! {
+    #![proptest_config(ProptestConfig {
+      cases: 1024, .. ProptestConfig::default()
+    })]
+    #[test]
+    // check that the two operation sequences have the same effect, enforcing the invariant of
+    // the transform function.
+    fn transform_invariant_holds(o1 in operation_strategy(), o2 in operation_strategy()) {
+        let (o1p, o2p) = Operation::transform(o1.clone(), o2.clone());
+
+        let mut db1 = newdb();
+        let mut db2 = newdb();
+
+        // Ensure that any expected tasks already exist
+        if let Operation::Update{ ref uuid, .. } = o1 {
+            let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
+            let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
+        }
+
+        if let Operation::Update{ ref uuid, .. } = o2 {
+            let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
+            let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
+        }
+
+        if let Operation::Delete{ ref uuid } = o1 {
+            let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
+            let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
+        }
+
+        if let Operation::Delete{ ref uuid } = o2 {
+            let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
+            let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
+        }
+
+        // if applying the initial operations fail, that indicates the operation was invalid
+        // in the base state, so consider the case successful.
+        if let Err(_) = db1.apply(o1) {
+            return Ok(());
+        }
+        if let Err(_) = db2.apply(o2) {
+            return Ok(());
+        }
+
+        if let Some(o) = o2p {
+            db1.apply(o).map_err(|e| TestCaseError::Fail(format!("Applying to db1: {}", e).into()))?;
+        }
+        if let Some(o) = o1p {
+            db2.apply(o).map_err(|e| TestCaseError::Fail(format!("Applying to db2: {}", e).into()))?;
+        }
+        assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
+    }
+}

taskchampion/tests/shared/mod.rs

@@ -0,0 +1,3 @@
+mod testserver;
+
+pub use testserver::TestServer;

taskchampion/tests/shared/testserver.rs

@@ -0,0 +1,81 @@
+use std::collections::HashMap;
+use taskchampion::server::{Blob, Server, VersionAdd};
+
+pub struct TestServer {
+    users: HashMap<String, User>,
+}
+
+struct User {
+    // versions, indexed at v-1
+    versions: Vec<Blob>,
+    snapshots: HashMap<u64, Blob>,
+}
+
+impl TestServer {
+    pub fn new() -> TestServer {
+        TestServer {
+            users: HashMap::new(),
+        }
+    }
+
+    fn get_user_mut(&mut self, username: &str) -> &mut User {
+        self.users
+            .entry(username.to_string())
+            .or_insert_with(User::new)
+    }
+}
+
+impl Server for TestServer {
+    /// Get a vector of all versions after `since_version`
+    fn get_versions(&self, username: &str, since_version: u64) -> Vec<Blob> {
+        self.users
+            .get(username)
+            .map(|user| user.get_versions(since_version))
+            .unwrap_or_else(|| vec![])
+    }
+
+    /// Add a new version.  If the given version number is incorrect, this responds with the
+    /// appropriate version and expects the caller to try again.
+    fn add_version(&mut self, username: &str, version: u64, blob: Blob) -> VersionAdd {
+        self.get_user_mut(username).add_version(version, blob)
+    }
+
+    fn add_snapshot(&mut self, username: &str, version: u64, blob: Blob) {
+        self.get_user_mut(username).add_snapshot(version, blob);
+    }
+}
+
+impl User {
+    fn new() -> User {
+        User {
+            versions: vec![],
+            snapshots: HashMap::new(),
+        }
+    }
+
+    fn get_versions(&self, since_version: u64) -> Vec<Blob> {
+        let last_version = self.versions.len();
+        if last_version == since_version as usize {
+            return vec![];
+        }
+        self.versions[since_version as usize..last_version]
+            .iter()
+            .map(|r| r.clone())
+            .collect::<Vec<Blob>>()
+    }
+
+    fn add_version(&mut self, version: u64, blob: Blob) -> VersionAdd {
+        // of by one here: client wants to send version 1 first
+        let expected_version = self.versions.len() as u64 + 1;
+        if version != expected_version {
+            return VersionAdd::ExpectedVersion(expected_version);
+        }
+        self.versions.push(blob);
+
+        VersionAdd::Ok
+    }
+
+    fn add_snapshot(&mut self, version: u64, blob: Blob) {
+        self.snapshots.insert(version, blob);
+    }
+}

taskchampion/tests/sync.rs

@@ -0,0 +1,123 @@
+use chrono::Utc;
+use taskchampion::{taskstorage, Operation, DB};
+use uuid::Uuid;
+
+mod shared;
+use shared::TestServer;
+
+fn newdb() -> DB {
+    DB::new(Box::new(taskstorage::InMemoryStorage::new()))
+}
+
+#[test]
+fn test_sync() {
+    let mut server = TestServer::new();
+
+    let mut db1 = newdb();
+    db1.sync("me", &mut server).unwrap();
+
+    let mut db2 = newdb();
+    db2.sync("me", &mut server).unwrap();
+
+    // make some changes in parallel to db1 and db2..
+    let uuid1 = Uuid::new_v4();
+    db1.apply(Operation::Create { uuid: uuid1 }).unwrap();
+    db1.apply(Operation::Update {
+        uuid: uuid1,
+        property: "title".into(),
+        value: Some("my first task".into()),
+        timestamp: Utc::now(),
+    })
+    .unwrap();
+
+    let uuid2 = Uuid::new_v4();
+    db2.apply(Operation::Create { uuid: uuid2 }).unwrap();
+    db2.apply(Operation::Update {
+        uuid: uuid2,
+        property: "title".into(),
+        value: Some("my second task".into()),
+        timestamp: Utc::now(),
+    })
+    .unwrap();
+
+    // and synchronize those around
+    db1.sync("me", &mut server).unwrap();
+    db2.sync("me", &mut server).unwrap();
+    db1.sync("me", &mut server).unwrap();
+    assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
+
+    // now make updates to the same task on both sides
+    db1.apply(Operation::Update {
+        uuid: uuid2,
+        property: "priority".into(),
+        value: Some("H".into()),
+        timestamp: Utc::now(),
+    })
+    .unwrap();
+    db2.apply(Operation::Update {
+        uuid: uuid2,
+        property: "project".into(),
+        value: Some("personal".into()),
+        timestamp: Utc::now(),
+    })
+    .unwrap();
+
+    // and synchronize those around
+    db1.sync("me", &mut server).unwrap();
+    db2.sync("me", &mut server).unwrap();
+    db1.sync("me", &mut server).unwrap();
+    assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
+}
+
+#[test]
+fn test_sync_create_delete() {
+    let mut server = TestServer::new();
+
+    let mut db1 = newdb();
+    db1.sync("me", &mut server).unwrap();
+
+    let mut db2 = newdb();
+    db2.sync("me", &mut server).unwrap();
+
+    // create and update a task..
+    let uuid = Uuid::new_v4();
+    db1.apply(Operation::Create { uuid }).unwrap();
+    db1.apply(Operation::Update {
+        uuid: uuid,
+        property: "title".into(),
+        value: Some("my first task".into()),
+        timestamp: Utc::now(),
+    })
+    .unwrap();
+
+    // and synchronize those around
+    db1.sync("me", &mut server).unwrap();
+    db2.sync("me", &mut server).unwrap();
+    db1.sync("me", &mut server).unwrap();
+    assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
+
+    // delete and re-create the task on db1
+    db1.apply(Operation::Delete { uuid }).unwrap();
+    db1.apply(Operation::Create { uuid }).unwrap();
+    db1.apply(Operation::Update {
+        uuid: uuid,
+        property: "title".into(),
+        value: Some("my second task".into()),
+        timestamp: Utc::now(),
+    })
+    .unwrap();
+
+    // and on db2, update a property of the task
+    db2.apply(Operation::Update {
+        uuid: uuid,
+        property: "project".into(),
+        value: Some("personal".into()),
+        timestamp: Utc::now(),
+    })
+    .unwrap();
+
+    db1.sync("me", &mut server).unwrap();
+    db2.sync("me", &mut server).unwrap();
+    db1.sync("me", &mut server).unwrap();
+    assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
+}

taskchampion/tests/sync_action_sequences.rs

@@ -0,0 +1,72 @@
+use chrono::Utc;
+use proptest::prelude::*;
+use taskchampion::{taskstorage, Operation, DB};
+use uuid::Uuid;
+
+mod shared;
+use shared::TestServer;
+
+fn newdb() -> DB {
+    DB::new(Box::new(taskstorage::InMemoryStorage::new()))
+}
+
+#[derive(Debug)]
+enum Action {
+    Op(Operation),
+    Sync,
+}
+
+fn action_sequence_strategy() -> impl Strategy<Value = Vec<(Action, u8)>> {
+    // Create, Update, Delete, or Sync on client 1, 2, .., followed by a round of syncs
+    "([CUDS][123])*S1S2S3S1S2".prop_map(|seq| {
+        let uuid = Uuid::parse_str("83a2f9ef-f455-4195-b92e-a54c161eebfc").unwrap();
+        seq.as_bytes()
+            .chunks(2)
+            .map(|action_on| {
+                let action = match action_on[0] {
+                    b'C' => Action::Op(Operation::Create { uuid }),
+                    b'U' => Action::Op(Operation::Update {
+                        uuid,
+                        property: "title".into(),
+                        value: Some("foo".into()),
+                        timestamp: Utc::now(),
+                    }),
+                    b'D' => Action::Op(Operation::Delete { uuid }),
+                    b'S' => Action::Sync,
+                    _ => unreachable!(),
+                };
+                let acton = action_on[1] - b'1';
+                (action, acton)
+            })
+            .collect::<Vec<(Action, u8)>>()
+    })
+}
+
+proptest! {
+    #[test]
+    // check that various sequences of operations on mulitple db's do not get the db's into an
+    // incompatible state.  The main concern here is that there might be a sequence of create
+    // and delete operations that results in a task existing in one DB but not existing in
+    // another.  So, the generated sequences focus on a single task UUID.
+    fn transform_sequences_of_operations(action_sequence in action_sequence_strategy()) {
+        let mut server = TestServer::new();
+        let mut dbs = [newdb(), newdb(), newdb()];
+
+        for (action, db) in action_sequence {
+            println!("{:?} on db {}", action, db);
+
+            let db = &mut dbs[db as usize];
+            match action {
+                Action::Op(op) => {
+                    if let Err(e) = db.apply(op) {
+                        println!("  {:?} (ignored)", e);
+                    }
+                },
+                Action::Sync => db.sync("me", &mut server).unwrap(),
+            }
+        }
+
+        assert_eq!(dbs[0].sorted_tasks(), dbs[0].sorted_tasks());
+        assert_eq!(dbs[1].sorted_tasks(), dbs[2].sorted_tasks());
+    }
+}