add WIP migration script

src/document_storage/migrate.clj

+(ns document-storage.migrate
+  (:require [document-storage.core :as ds]
+            [document-storage.postgres.core :as dspostgres]
+            [environ.core :refer [env]]
+            [clojure.tools.logging :as log]
+            [editscript.core :as escript]))
+
+
+(defn migrate-repository [repo-id ds-source ds-target]
+  (let [documents (.list-documents ds-source repo-id)]
+    (doseq [doc-id documents]
+      (log/info "[" repo-id "] Start Migrating " doc-id)
+      ;;;TODO check if repository is versioned
+      (let [target-doc (.load-document ds-target doc-id repo-id)
+            source-doc (.load-document ds-source doc-id repo-id)]
+        (if (and target-doc
+                 (= target-doc
+                    source-doc))
+          (log/info "[" repo-id "] Document already migrated " doc-id)
+          (if (.versioned-repository? ds-source repo-id)
+            (let [versions (sort-by (comp #(. % getTime) :date) < (.list-versions ds-source doc-id repo-id))]
+              (doseq [dv versions]
+                (if (= (:type dv) :delete)
+                  (log/info "[" repo-id "] Document " doc-id " - Version: " (:version dv) " - Type: DELETE")
+                  (let [source-document (if (= (count versions)
+                                               1)
+                                          (.load-document ds-source doc-id repo-id)
+                                          (.load-version ds-source doc-id (:version dv) repo-id))]
+                    (log/info "[" repo-id "] Save Document " doc-id " - Version: " (:version dv))
+                    (.save-document ds-target doc-id source-document repo-id #_{:force-keyframe true})))))
+            (.save-document ds-target doc-id source-doc repo-id))))
+      (log/info "[" repo-id "] Finished Migrating " doc-id))))
+
+(defn migrate [ds-source ds-destination]
+  (let [repositories (.list-repositories ds-source)]
+    (doall
+     (pmap
+      (fn [r]
+        (do (log/info "Migrate Repository " (:id r))
+            (migrate-repository (:id r) ds-source ds-destination)
+            (log/info "Finished Migrating " (:id r))))
+      repositories))))
+
+
+#_(def gitstorage (ds/make-git-ds "doc-storage"))
+#_(def pgstorage (dspostgres/make-postgres-ds
+                {:id :mtesting
+                 :repositories [:atoms :courselists
+                                :courses
+                                :default
+                                :exams
+                                :exam-sessions
+                                :linklists
+                                :live-surveys
+                                :quiz-sessions
+                                :quizzes
+                                :test
+                                :units
+                                :unit-sessions]
+                 :jdbc-url (env :database-url)}))
+
+#_(.list-repositories gitstorage)
+
+#_(.versioned-repository? gitstorage :atoms)
+#_(time (do (.load-document pgstorage "20210304T190941-uc82822" :units)
+            (+ 1 1)
+            ))
+
+(time (do (.load-document gitstorage "q2018-10-06T11-03-59-G__1589" :quizzes)
+          (+ 1 1)))
+
+(time (doseq [d ["q2018-04-19T11-26-06-G__64819"
+                 "q2020-11-09T08-06-25-G__486"
+                 "q2018-04-19T11-26-05-G__64811"
+                 "q2018-04-19T11-26-06-G__64817"
+                 "q2020-11-09T09-24-36-G__489"
+                 "q2018-04-19T11-26-07-G__64820"
+                 "q2018-04-19T11-26-07-G__64821"
+                 "q2018-04-19T11-26-06-G__64816"
+                 "q2018-04-19T11-26-10-G__64823"
+                 "q2018-04-19T11-26-04-G__64808"
+                 "q2018-04-19T11-26-10-G__64824"
+                 "q2020-11-09T08-06-05-G__294"
+                 "q2018-04-19T11-26-05-G__64812"
+                 "q2020-11-09T08-08-49-G__1000"
+                 "q2018-04-19T11-26-09-G__64822"
+                 "q2018-04-19T11-26-04-G__64809"
+                 "q2018-05-23T20-35-52-G__1527"
+                 "q2018-04-19T11-26-06-G__64818"]]
+        (.load-document pgstorage d :quizzes)
+        (* 1 1)))
+
+
+
+#_(let [previous-version (.load-version gitstorage "20210927T120641-atom80156" "ca202f62c216956095d483884ff0641b63ee0b91" :atoms)
+      new-version (.load-version gitstorage "20210927T120641-atom80156" "001053e57e989041887bddd07672ec9e6cc7d6fb" :atoms)]
+  #_(.save-document pgstorage "20210927T120641-atom80156" new-version "atoms")
+  #_previous-version
+  (escript/diff previous-version new-version))
+
+#_(let [previous-version (.load-version gitstorage "20210927T120641-atom80156" "ca202f62c216956095d483884ff0641b63ee0b91" :atoms)
+      previous-pg-version (.load-document pgstorage "20210927T120641-atom80156" :atoms)
+      new-version (.load-version gitstorage "20210927T120641-atom80156" "001053e57e989041887bddd07672ec9e6cc7d6fb" :atoms)]
+  #_(.save-document pgstorage "20210927T120641-atom80156" new-version "atoms")
+  #_previous-version
+  #_(= previous-version previous-pg-version)
+    ;;;FIXME this will crash with a-start?!
+  #_(escript/diff previous-pg-version new-version {:str-diff? true :algo :a-star})
+
+  previous-pg-version
+  )
+#_(time (.load-document pgstorage "u2019-01-14T22-08-05-G__417" :units))
+
+#_(time (.load-version pgstorage "u2019-01-14T22-08-05-G__417"  160 :units))
+
+
+
+#_(= (.load-version gitstorage "20180926T110658-atom554" "ebc38a8856a2a86a5d6cde415824ce0300fbff64" #_"4357f94291c0e2e2df758f1fd61c1c06f9ce7bbf" :atoms)
+   (.load-document gitstorage "20180926T110658-atom554" :atoms))
+#_(last (sort-by (comp #(. % getTime) :date) < (.list-versions gitstorage "20180926T110658-atom554" :atoms)))
+#_(map (comp println :commit) (sort-by (comp #(. % getTime) :date) < (.list-versions gitstorage "20180926T110658-atom554" :atoms)))
+
+#_(sort-by (comp #(. % getTime) :date) < (.list-versions gitstorage "20180926T110658-atom554" :atoms))
+
+#_(migrate gitstorage pgstorage)
+#_(.list-documents gitstorage :atoms)
+#_(let [repo (first (.list-repositories gitstorage))
+      repo-id (:id repo)
+      doc (nth (.list-documents gitstorage repo-id) 16)
+      versions (.list-versions gitstorage doc repo-id)
+      version (first versions)]
+  (.load-version gitstorage doc (:version version) repo-id)
+  )
+
+#_((comp #(. % getTime) :time) {:time (new java.util.Date)})
+
+
+
+;;;FIXME Error when loading the version after this version
+;;; 21-11-27 21:30:47 fedora INFO [document-storage.migrate:326] - [ :atoms ] Save Document  20180926T110658-atom554  - Version:  4357f94291c0e2e2df758f1fd61c1c06f9ce7bbf
+;;; Execution error (NullPointerException) at org.eclipse.jgit.internal.storage.file.UnpackedObjectCache$Table/index (UnpackedObjectCache.java:146).
+;;; null
+
+
+;;;FIXME log for deleted file or something else?
+;; 21-11-28 11:53:25 fedora INFO [document-storage.migrate:326] - [ :atoms ] Save Document  brince  - Version:  2fe708e3e2c9964d87f086dbf6cd838bfd3523a5
+;; 21-11-28 11:53:26 fedora INFO [document-storage.migrate:326] - [ :units ] Save Document  u2020-04-20T08-14-21-G__67  - Version:  f7b882a7640dd8ba1c8e9ee3020b6fded5bd9729
+;; 21-11-28 11:53:27 fedora INFO [document-storage.migrate:326] - [ :atoms ] Save Document  prince  - Version:  c1d9d886d76a08892cb47176380f013a649b1f0e
+;; Execution error (NoHeadException) at org.eclipse.jgit.api.LogCommand/call (LogCommand.java:150).
+;; No HEAD exists and no explicit starting revision was specified
+
+
+
+;;;TODO SELECT RELEVANT FRAMES
+
+;; with RECURSIVE rec AS (SELECT ru.document_id, ru.id, ru.data_type, ru.version, 0 AS degree, reference
+;;         FROM repo_units ru, (select rr.id from repo_units rr where document_id = 'u2020-04-08T14-35-53-G__1513' order by version desc limit 1) as init
+;;         WHERE ru.id  = init.id
+;;         UNION
+;;         SELECT ru.document_id, ru.id, ru.data_type ,ru. "version", rec.degree + 1 AS degree, ru.reference
+;;         FROM repo_units ru
+;;         JOIN rec ON ru.id = rec.reference and rec.data_type != 'key')
+;; select * from rec;