Skip to content
Snippets Groups Projects
Commit be5d58ca authored by Bruno Burke's avatar Bruno Burke :hamburger:
Browse files

add WIP migration script

parent 27d4e1a5
No related branches found
No related tags found
1 merge request!1Draft: Migration
Pipeline #93687 passed
(ns document-storage.migrate
(:require [document-storage.core :as ds]
[document-storage.postgres.core :as dspostgres]
[environ.core :refer [env]]
[clojure.tools.logging :as log]
[editscript.core :as escript]))
(defn migrate-repository [repo-id ds-source ds-target]
(let [documents (.list-documents ds-source repo-id)]
(doseq [doc-id documents]
(log/info "[" repo-id "] Start Migrating " doc-id)
;;;TODO check if repository is versioned
(let [target-doc (.load-document ds-target doc-id repo-id)
source-doc (.load-document ds-source doc-id repo-id)]
(if (and target-doc
(= target-doc
source-doc))
(log/info "[" repo-id "] Document already migrated " doc-id)
(if (.versioned-repository? ds-source repo-id)
(let [versions (sort-by (comp #(. % getTime) :date) < (.list-versions ds-source doc-id repo-id))]
(doseq [dv versions]
(if (= (:type dv) :delete)
(log/info "[" repo-id "] Document " doc-id " - Version: " (:version dv) " - Type: DELETE")
(let [source-document (if (= (count versions)
1)
(.load-document ds-source doc-id repo-id)
(.load-version ds-source doc-id (:version dv) repo-id))]
(log/info "[" repo-id "] Save Document " doc-id " - Version: " (:version dv))
(.save-document ds-target doc-id source-document repo-id #_{:force-keyframe true})))))
(.save-document ds-target doc-id source-doc repo-id))))
(log/info "[" repo-id "] Finished Migrating " doc-id))))
(defn migrate [ds-source ds-destination]
(let [repositories (.list-repositories ds-source)]
(doall
(pmap
(fn [r]
(do (log/info "Migrate Repository " (:id r))
(migrate-repository (:id r) ds-source ds-destination)
(log/info "Finished Migrating " (:id r))))
repositories))))
#_(def gitstorage (ds/make-git-ds "doc-storage"))
#_(def pgstorage (dspostgres/make-postgres-ds
{:id :mtesting
:repositories [:atoms :courselists
:courses
:default
:exams
:exam-sessions
:linklists
:live-surveys
:quiz-sessions
:quizzes
:test
:units
:unit-sessions]
:jdbc-url (env :database-url)}))
#_(.list-repositories gitstorage)
#_(.versioned-repository? gitstorage :atoms)
#_(time (do (.load-document pgstorage "20210304T190941-uc82822" :units)
(+ 1 1)
))
(time (do (.load-document gitstorage "q2018-10-06T11-03-59-G__1589" :quizzes)
(+ 1 1)))
(time (doseq [d ["q2018-04-19T11-26-06-G__64819"
"q2020-11-09T08-06-25-G__486"
"q2018-04-19T11-26-05-G__64811"
"q2018-04-19T11-26-06-G__64817"
"q2020-11-09T09-24-36-G__489"
"q2018-04-19T11-26-07-G__64820"
"q2018-04-19T11-26-07-G__64821"
"q2018-04-19T11-26-06-G__64816"
"q2018-04-19T11-26-10-G__64823"
"q2018-04-19T11-26-04-G__64808"
"q2018-04-19T11-26-10-G__64824"
"q2020-11-09T08-06-05-G__294"
"q2018-04-19T11-26-05-G__64812"
"q2020-11-09T08-08-49-G__1000"
"q2018-04-19T11-26-09-G__64822"
"q2018-04-19T11-26-04-G__64809"
"q2018-05-23T20-35-52-G__1527"
"q2018-04-19T11-26-06-G__64818"]]
(.load-document pgstorage d :quizzes)
(* 1 1)))
#_(let [previous-version (.load-version gitstorage "20210927T120641-atom80156" "ca202f62c216956095d483884ff0641b63ee0b91" :atoms)
new-version (.load-version gitstorage "20210927T120641-atom80156" "001053e57e989041887bddd07672ec9e6cc7d6fb" :atoms)]
#_(.save-document pgstorage "20210927T120641-atom80156" new-version "atoms")
#_previous-version
(escript/diff previous-version new-version))
#_(let [previous-version (.load-version gitstorage "20210927T120641-atom80156" "ca202f62c216956095d483884ff0641b63ee0b91" :atoms)
previous-pg-version (.load-document pgstorage "20210927T120641-atom80156" :atoms)
new-version (.load-version gitstorage "20210927T120641-atom80156" "001053e57e989041887bddd07672ec9e6cc7d6fb" :atoms)]
#_(.save-document pgstorage "20210927T120641-atom80156" new-version "atoms")
#_previous-version
#_(= previous-version previous-pg-version)
;;;FIXME this will crash with a-start?!
#_(escript/diff previous-pg-version new-version {:str-diff? true :algo :a-star})
previous-pg-version
)
#_(time (.load-document pgstorage "u2019-01-14T22-08-05-G__417" :units))
#_(time (.load-version pgstorage "u2019-01-14T22-08-05-G__417" 160 :units))
#_(= (.load-version gitstorage "20180926T110658-atom554" "ebc38a8856a2a86a5d6cde415824ce0300fbff64" #_"4357f94291c0e2e2df758f1fd61c1c06f9ce7bbf" :atoms)
(.load-document gitstorage "20180926T110658-atom554" :atoms))
#_(last (sort-by (comp #(. % getTime) :date) < (.list-versions gitstorage "20180926T110658-atom554" :atoms)))
#_(map (comp println :commit) (sort-by (comp #(. % getTime) :date) < (.list-versions gitstorage "20180926T110658-atom554" :atoms)))
#_(sort-by (comp #(. % getTime) :date) < (.list-versions gitstorage "20180926T110658-atom554" :atoms))
#_(migrate gitstorage pgstorage)
#_(.list-documents gitstorage :atoms)
#_(let [repo (first (.list-repositories gitstorage))
repo-id (:id repo)
doc (nth (.list-documents gitstorage repo-id) 16)
versions (.list-versions gitstorage doc repo-id)
version (first versions)]
(.load-version gitstorage doc (:version version) repo-id)
)
#_((comp #(. % getTime) :time) {:time (new java.util.Date)})
;;;FIXME Error when loading the version after this version
;;; 21-11-27 21:30:47 fedora INFO [document-storage.migrate:326] - [ :atoms ] Save Document 20180926T110658-atom554 - Version: 4357f94291c0e2e2df758f1fd61c1c06f9ce7bbf
;;; Execution error (NullPointerException) at org.eclipse.jgit.internal.storage.file.UnpackedObjectCache$Table/index (UnpackedObjectCache.java:146).
;;; null
;;;FIXME log for deleted file or something else?
;; 21-11-28 11:53:25 fedora INFO [document-storage.migrate:326] - [ :atoms ] Save Document brince - Version: 2fe708e3e2c9964d87f086dbf6cd838bfd3523a5
;; 21-11-28 11:53:26 fedora INFO [document-storage.migrate:326] - [ :units ] Save Document u2020-04-20T08-14-21-G__67 - Version: f7b882a7640dd8ba1c8e9ee3020b6fded5bd9729
;; 21-11-28 11:53:27 fedora INFO [document-storage.migrate:326] - [ :atoms ] Save Document prince - Version: c1d9d886d76a08892cb47176380f013a649b1f0e
;; Execution error (NoHeadException) at org.eclipse.jgit.api.LogCommand/call (LogCommand.java:150).
;; No HEAD exists and no explicit starting revision was specified
;;;TODO SELECT RELEVANT FRAMES
;; with RECURSIVE rec AS (SELECT ru.document_id, ru.id, ru.data_type, ru.version, 0 AS degree, reference
;; FROM repo_units ru, (select rr.id from repo_units rr where document_id = 'u2020-04-08T14-35-53-G__1513' order by version desc limit 1) as init
;; WHERE ru.id = init.id
;; UNION
;; SELECT ru.document_id, ru.id, ru.data_type ,ru. "version", rec.degree + 1 AS degree, ru.reference
;; FROM repo_units ru
;; JOIN rec ON ru.id = rec.reference and rec.data_type != 'key')
;; select * from rec;
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment