Skip to content
Snippets Groups Projects
Commit f8e1bb8a authored by Bruno Burke's avatar Bruno Burke :hamburger:
Browse files

update migrate.clj

parent a716f43a
No related branches found
No related tags found
1 merge request!1Draft: Migration
Pipeline #95830 failed
Stage: build
Stage: test
Stage: visualize
Hide whitespace changes
Inline Side-by-side
src/document_storage/migrate.clj
+ 114
− 123
View file @ f8e1bb8a
...@@ -7,108 +7,128 @@ ...@@ -7,108 +7,128 @@
(defn migrate-repository [repo-id ds-source ds-target] (defn migrate-repository [repo-id ds-source ds-target]
(let [documents (.list-documents ds-source repo-id)] ;;;TODO git is slow, load all git document versions in parallel
(doseq [doc-id documents] (let [documents (map-indexed vector (.list-documents ds-source repo-id))
(log/info "[" repo-id "] Start Migrating " doc-id) doc-count (count documents)]
(doall
(pmap (fn [[i doc-id]]
(let [dinfo (str "[" repo-id "][" i "/" doc-count "]")]
(log/info dinfo "Start Migrating " doc-id)
;;;TODO check if repository is versioned ;;;TODO check if repository is versioned
(let [target-doc (.load-document ds-target doc-id repo-id) (let [target-doc (.load-document ds-target doc-id repo-id)
source-doc (.load-document ds-source doc-id repo-id)] source-doc (.load-document ds-source doc-id repo-id)]
(if (and target-doc (if (and target-doc
(= target-doc (= target-doc
source-doc)) source-doc))
(log/info "[" repo-id "] Document already migrated " doc-id) (log/info dinfo "Document already migrated " doc-id)
(if (.versioned-repository? ds-source repo-id) (if (.versioned-repository? ds-source repo-id)
(let [versions (sort-by (comp #(. % getTime) :date) < (.list-versions ds-source doc-id repo-id))] (let [versions (sort-by (comp #(. % getTime) :date) < (.list-versions ds-source doc-id repo-id))]
(doseq [dv versions] (doseq [dv versions]
(if (= (:type dv) :delete) (if (= (:type dv) :delete)
(log/info "[" repo-id "] Document " doc-id " - Version: " (:version dv) " - Type: DELETE") (do
(let [source-document (if (= (count versions) (log/info dinfo "Document " doc-id " - Version: " (:version dv) " - Type: DELETE")
1) (.delete-document ds-target doc-id repo-id))
(.load-document ds-source doc-id repo-id) (let [source-document (if (= (count versions) 1)
(.load-version ds-source doc-id (:version dv) repo-id))] source-doc
(log/info "[" repo-id "] Save Document " doc-id " - Version: " (:version dv)) (.load-version ds-source doc-id (:version dv) repo-id))]
(.save-document ds-target doc-id source-document repo-id #_{:force-keyframe true}))))) (log/info dinfo "Save Document " doc-id " - Version: " (:version dv))
(.save-document ds-target doc-id source-doc repo-id)))) (.save-document ds-target doc-id source-document repo-id #_{:force-keyframe true})
(log/info "[" repo-id "] Finished Migrating " doc-id)))) (if (= source-document (.load-document ds-target doc-id repo-id))
(log/info dinfo "Check Document " doc-id " - Version: " (:version dv) " passed")
(log/error dinfo "Check Document " doc-id " - Version: " (:version dv) " failed"))))))
(do
(.save-document ds-target doc-id source-doc repo-id)
(if (= source-doc (.load-document ds-target doc-id repo-id))
(log/info dinfo "Check Document " doc-id " passed")
(log/error dinfo "Check Document " doc-id " failed"))))))
(log/info dinfo "Finished Migrating " doc-id))) documents))))
(defn migrate [ds-source ds-destination] (defn migrate [ds-source ds-destination]
(let [repositories (.list-repositories ds-source)] (let [repositories (.list-repositories ds-source)]
(doall (doseq [r repositories]
(pmap (log/info "Migrate Repository " (:id r))
(fn [r] (migrate-repository (:id r) ds-source ds-destination)
(do (log/info "Migrate Repository " (:id r)) (log/info "Finished Migrating " (:id r)))))
(migrate-repository (:id r) ds-source ds-destination)
(log/info "Finished Migrating " (:id r))))
repositories))))
#_(def gitstorage (ds/make-git-ds "doc-storage")) #_(def gitstorage (ds/make-git-ds "doc-storage"))
#_(def pgstorage (dspostgres/make-postgres-ds #_(def pgstorage (dspostgres/make-postgres-ds
{:id :mtesting {:id :mtesting
:repositories [:atoms :courselists :repositories {:atoms {:caching true}
:courses :courselists {:caching true}
:default :courses {:caching true}
:exams :default {:caching true}
:exam-sessions :exams {:caching true}
:linklists :exam-sessions {}
:live-surveys :linklists {:versioning false}
:quiz-sessions :live-surveys {:versioning false}
:quizzes :quiz-sessions {}
:test :quizzes {:caching true}
:units :test {:caching true}
:unit-sessions] :units {:caching true}
:jdbc-url (env :database-url)})) :unit-sessions {:versioning false}}
:jdbc-url (env :database-url)}))
#_(time (migrate gitstorage pgstorage))
;;;TODO add caching for pgstorage
#_(.list-repositories gitstorage) #_(.list-repositories gitstorage)
#_(.list-repositories pgstorage)
#_(.versioned-repository? gitstorage :atoms) #_(document-storage.postgres.documents/get-last-document-version pgstorage "" :atoms)
#_(.versioned-repository? pgstorage :atoms)
#_(.cached-repository? pgstorage :linklists)
#_(time (do (.load-document pgstorage "20210304T190941-uc82822" :units) #_(time (do (.load-document pgstorage "20210304T190941-uc82822" :units)
(+ 1 1) (+ 1 1)))
))
#_(time (do (.load-document pgstorage "q2018-10-06T11-03-59-G__1589" :quizzes)
(time (do (.load-document gitstorage "q2018-10-06T11-03-59-G__1589" :quizzes) (+ 1 1)))
(+ 1 1)))
#_(defn test-version [storage id repository]
(time (doseq [d ["q2018-04-19T11-26-06-G__64819" (let [version (first (take-last 2 (.list-versions storage id repository)))]
"q2020-11-09T08-06-25-G__486" (.load-version storage id (:version version) repository)))
"q2018-04-19T11-26-05-G__64811"
"q2018-04-19T11-26-06-G__64817" #_(time (test-version gitstorage "q2018-10-06T11-03-59-G__1589" :quizzes))
"q2020-11-09T09-24-36-G__489" #_(time (test-version pgstorage "q2018-10-06T11-03-59-G__1589" :quizzes))
"q2018-04-19T11-26-07-G__64820" #_(time (doseq [d ["q2018-04-19T11-26-06-G__64819"
"q2018-04-19T11-26-07-G__64821" "q2020-11-09T08-06-25-G__486"
"q2018-04-19T11-26-06-G__64816" "q2018-04-19T11-26-05-G__64811"
"q2018-04-19T11-26-10-G__64823" "q2018-04-19T11-26-06-G__64817"
"q2018-04-19T11-26-04-G__64808" "q2020-11-09T09-24-36-G__489"
"q2018-04-19T11-26-10-G__64824" "q2018-04-19T11-26-07-G__64820"
"q2020-11-09T08-06-05-G__294" "q2018-04-19T11-26-07-G__64821"
"q2018-04-19T11-26-05-G__64812" "q2018-04-19T11-26-06-G__64816"
"q2020-11-09T08-08-49-G__1000" "q2018-04-19T11-26-10-G__64823"
"q2018-04-19T11-26-09-G__64822" "q2018-04-19T11-26-04-G__64808"
"q2018-04-19T11-26-04-G__64809" "q2018-04-19T11-26-10-G__64824"
"q2018-05-23T20-35-52-G__1527" "q2020-11-09T08-06-05-G__294"
"q2018-04-19T11-26-06-G__64818"]] "q2018-04-19T11-26-05-G__64812"
(.load-document pgstorage d :quizzes) "q2020-11-09T08-08-49-G__1000"
(* 1 1))) "q2018-04-19T11-26-09-G__64822"
"q2018-04-19T11-26-04-G__64809"
"q2018-05-23T20-35-52-G__1527"
"q2018-04-19T11-26-06-G__64818"]]
(.load-document pgstorage d :quizzes)
(* 1 1)))
#_(time (do (.load-document gitstorage "u2019-01-14T22-08-05-G__417" :units)
(+ 11 1)))
#_(let [previous-version (.load-version gitstorage "20210927T120641-atom80156" "ca202f62c216956095d483884ff0641b63ee0b91" :atoms) #_(let [previous-version (.load-version gitstorage "20210927T120641-atom80156" "ca202f62c216956095d483884ff0641b63ee0b91" :atoms)
new-version (.load-version gitstorage "20210927T120641-atom80156" "001053e57e989041887bddd07672ec9e6cc7d6fb" :atoms)] new-version (.load-version gitstorage "20210927T120641-atom80156" "001053e57e989041887bddd07672ec9e6cc7d6fb" :atoms)]
#_(.save-document pgstorage "20210927T120641-atom80156" new-version "atoms") #_(.save-document pgstorage "20210927T120641-atom80156" new-version "atoms")
#_previous-version #_previous-version
(escript/diff previous-version new-version)) (escript/diff previous-version new-version))
#_(let [previous-version (.load-version gitstorage "20210927T120641-atom80156" "ca202f62c216956095d483884ff0641b63ee0b91" :atoms) #_(let [previous-version (.load-version gitstorage "20210927T120641-atom80156" "ca202f62c216956095d483884ff0641b63ee0b91" :atoms)
previous-pg-version (.load-document pgstorage "20210927T120641-atom80156" :atoms) previous-pg-version (.load-document pgstorage "20210927T120641-atom80156" :atoms)
new-version (.load-version gitstorage "20210927T120641-atom80156" "001053e57e989041887bddd07672ec9e6cc7d6fb" :atoms)] new-version (.load-version gitstorage "20210927T120641-atom80156" "001053e57e989041887bddd07672ec9e6cc7d6fb" :atoms)]
#_(.save-document pgstorage "20210927T120641-atom80156" new-version "atoms") #_(.save-document pgstorage "20210927T120641-atom80156" new-version "atoms")
#_previous-version #_previous-version
#_(= previous-version previous-pg-version) #_(= previous-version previous-pg-version)
;;;FIXME this will crash with a-start?! ;;;FIXME this will crash with a-star?!
#_(escript/diff previous-pg-version new-version {:str-diff? true :algo :a-star}) #_(escript/diff previous-pg-version new-version {:str-diff? true :algo :a-star})
previous-pg-version previous-pg-version)
)
#_(time (.load-document pgstorage "u2019-01-14T22-08-05-G__417" :units)) #_(time (.load-document pgstorage "u2019-01-14T22-08-05-G__417" :units))
#_(time (.load-version pgstorage "u2019-01-14T22-08-05-G__417" 160 :units)) #_(time (.load-version pgstorage "u2019-01-14T22-08-05-G__417" 160 :units))
...@@ -116,48 +136,19 @@ ...@@ -116,48 +136,19 @@
#_(= (.load-version gitstorage "20180926T110658-atom554" "ebc38a8856a2a86a5d6cde415824ce0300fbff64" #_"4357f94291c0e2e2df758f1fd61c1c06f9ce7bbf" :atoms) #_(= (.load-version gitstorage "20180926T110658-atom554" "ebc38a8856a2a86a5d6cde415824ce0300fbff64" #_"4357f94291c0e2e2df758f1fd61c1c06f9ce7bbf" :atoms)
(.load-document gitstorage "20180926T110658-atom554" :atoms)) (.load-document gitstorage "20180926T110658-atom554" :atoms))
#_(last (sort-by (comp #(. % getTime) :date) < (.list-versions gitstorage "20180926T110658-atom554" :atoms))) #_(last (sort-by (comp #(. % getTime) :date) < (.list-versions gitstorage "20180926T110658-atom554" :atoms)))
#_(map (comp println :commit) (sort-by (comp #(. % getTime) :date) < (.list-versions gitstorage "20180926T110658-atom554" :atoms))) #_(map (comp println :commit) (sort-by (comp #(. % getTime) :date) < (.list-versions gitstorage "20180926T110658-atom554" :atoms)))
#_(sort-by (comp #(. % getTime) :date) < (.list-versions gitstorage "20180926T110658-atom554" :atoms)) #_(sort-by (comp #(. % getTime) :date) < (.list-versions gitstorage "20180926T110658-atom554" :atoms))
#_(migrate gitstorage pgstorage)
#_(.list-documents gitstorage :atoms) #_(.list-documents gitstorage :atoms)
#_(let [repo (first (.list-repositories gitstorage)) #_(let [repo (first (.list-repositories gitstorage))
repo-id (:id repo) repo-id (:id repo)
doc (nth (.list-documents gitstorage repo-id) 16) doc (nth (.list-documents gitstorage repo-id) 16)
versions (.list-versions gitstorage doc repo-id) versions (.list-versions gitstorage doc repo-id)
version (first versions)] version (first versions)]
(.load-version gitstorage doc (:version version) repo-id) (.load-version gitstorage doc (:version version) repo-id))
)
#_((comp #(. % getTime) :time) {:time (new java.util.Date)}) #_((comp #(. % getTime) :time) {:time (new java.util.Date)})
;;;FIXME Error when loading the version after this version
;;; 21-11-27 21:30:47 fedora INFO [document-storage.migrate:326] - [ :atoms ] Save Document 20180926T110658-atom554 - Version: 4357f94291c0e2e2df758f1fd61c1c06f9ce7bbf
;;; Execution error (NullPointerException) at org.eclipse.jgit.internal.storage.file.UnpackedObjectCache$Table/index (UnpackedObjectCache.java:146).
;;; null
;;;FIXME log for deleted file or something else?
;; 21-11-28 11:53:25 fedora INFO [document-storage.migrate:326] - [ :atoms ] Save Document brince - Version: 2fe708e3e2c9964d87f086dbf6cd838bfd3523a5
;; 21-11-28 11:53:26 fedora INFO [document-storage.migrate:326] - [ :units ] Save Document u2020-04-20T08-14-21-G__67 - Version: f7b882a7640dd8ba1c8e9ee3020b6fded5bd9729
;; 21-11-28 11:53:27 fedora INFO [document-storage.migrate:326] - [ :atoms ] Save Document prince - Version: c1d9d886d76a08892cb47176380f013a649b1f0e
;; Execution error (NoHeadException) at org.eclipse.jgit.api.LogCommand/call (LogCommand.java:150).
;; No HEAD exists and no explicit starting revision was specified
;;;TODO SELECT RELEVANT FRAMES
;; with RECURSIVE rec AS (SELECT ru.document_id, ru.id, ru.data_type, ru.version, 0 AS degree, reference
;; FROM repo_units ru, (select rr.id from repo_units rr where document_id = 'u2020-04-08T14-35-53-G__1513' order by version desc limit 1) as init
;; WHERE ru.id = init.id
;; UNION
;; SELECT ru.document_id, ru.id, ru.data_type ,ru. "version", rec.degree + 1 AS degree, ru.reference
;; FROM repo_units ru
;; JOIN rec ON ru.id = rec.reference and rec.data_type != 'key')
;; select * from rec;
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment