(ns cmr.graph.demo.movie
  (:require
   [clojure.string :as string]
   [clojurewerkz.neocons.rest :as nr]
   [clojurewerkz.neocons.rest.cypher :as cy]
   [cmr.graph.queries.neo4j.demo.movie :as query]
   [taoensso.timbre :as log]))

(defn get-graph
  ([conn]
    (get-graph conn 100))
  ([conn limit]
    (when (string? limit)
      (get-graph conn (Integer/parseInt limit)))
    (let   [result (cy/tquery conn query/graph {:limit limit})
            nodes (map (fn [{:strs [cast movie]}]
                         (concat [{:title movie
                                   :label :movie}]
                                 (map (fn [x] {:title x
                                               :label :actor})
                                      cast)))
                       result)
            nodes (distinct (apply concat nodes))
            nodes-index (into {} (map-indexed #(vector %2 %1) nodes))
            links (map (fn [{:strs [cast movie]}]
                         (let [target   (nodes-index {:title movie :label :movie})]
                           (map (fn [x]
                                  {:target target
                                   :source (nodes-index {:title x :label :actor})})
                                       cast)))
                       result)]
    {:nodes nodes :links (flatten links)})))

(defn search
  [conn q]
  (if (string/blank? q)
    []
    (let  [result (cy/tquery conn query/search {:title (str "(?i).*" q ".*")})]
      (map (fn [x] {:movie (:data (x "movie"))}) result))))

(defn get-movie
  [conn title]
  (log/trace "Got connection:" conn)
  (log/trace "Got title:" title)
  (let [[result] (cy/tquery conn query/title {:title title})]
    result))

(ns cmr.graph.queries.neo4j.demo.movie)

(def graph "MATCH (m:Movie)<-[:ACTED_IN]-(a:Person)
            RETURN m.title as movie, collect(a.name) as cast
            LIMIT {limit};")

(def search "MATCH (movie:Movie) WHERE movie.title =~ {title} RETURN movie;")

(def title "MATCH (movie:Movie {title:{title}})
            OPTIONAL MATCH (movie)<-[r]-(person:Person)
            RETURN movie.title as title,
                   collect({name:person.name,
                            job:head(split(lower(type(r)),'_')),
                            role:r.roles}) as cast LIMIT 1;")

(ns cmr.graph.queries.neo4j.collections
  (:require
   [clojure.string :as string]))

(def reset "MATCH (n) DETACH DELETE n;")

(def get-all "MATCH (collection:Collection) RETURN collection;")
(def delete-all "MATCH (collection:Collection) DELETE collection;")
(def delete-all-cascade "MATCH (collection:Collection) DETACH DELETE collection;")

(defn get-urls-by-concept-id
  [concept-id]
  (format "match (c:Collection)-[:LINKS_TO]->(u:Url) where c.conceptId='%s' return u.name;"
          concept-id))

(defn get-concept-ids-by-urls
  [urls]
  (format "match (c:Collection)-[:LINKS_TO]->(u:Url) where u.name in [%s] return c.conceptId;"
          (string/join "," (map #(format "'%s'" %) urls))))

(ns cmr.graph.core
  (:require
   [clojusc.twig :as logger]
   [cmr.graph.components.core :as componemts]
   [com.stuartsierra.component :as component]
   [taoensso.timbre :as log]
   [trifl.java :as java])
  (:gen-class))

(def startup-message
  (str "Component startup complete."
       \newline \newline
       "The CMR Graph system is now ready to use. "
       "To get started,"
       \newline
       "you can visit the following:"
       \newline
       "* Neo4j: http://localhost:7474/browser/"
       \newline
       "* Kibana: http://localhost:5601/"
       \newline \newline
       "Additionally, the CMR Graph REST API is available at"
       \newline
       "http://localhost:3012. To try it out, call the following:"
       \newline
       "* curl --silent \"http://localhost:3012/ping\
       \newline
       "* curl --silent \"http://localhost:3012/health\
       \newline))

(defn shutdown
  [system]
  (component/stop system))

(defn -main
  [& args]
  (logger/set-level! ['cmr.graph] :info)
  (log/info "Starting the CMR Graph components ...")
  (let [system (componemts/init)]
    (component/start system)
    (java/add-shutdown-handler #(shutdown system)))
  (log/info startup-message))

(ns cmr.graph.system.impl.state
  (:require
    [com.stuartsierra.component :as component]
    [taoensso.timbre :as log]))

(def ^:dynamic *state*
  (atom {:status :stopped
         :system nil
         :ns }))

(defrecord StateTracker [])

(defn get-state
  [_this]
  @*state*)

(defn set-state
  [_this new-state]
  (reset! *state* new-state))

(defn get-status
  [this]
  (:status (get-state this)))

(defn set-status
  [this value]
  (set-state this (assoc (get-state this) :status value)))

(defn get-system
  [this]
  (:system (get-state this)))

(defn set-system
  [this value]
  (set-state this (assoc (get-state this) :system value)))

(defn get-system-ns
  [this]
  (:ns (get-state this)))

(defn set-system-ns
  [this an-ns]
  (set-state this (assoc (get-state this) :ns an-ns)))

(def behaviour
  {:get-state get-state
   :set-state set-state
   :get-status get-status
   :set-status set-status
   :get-system get-system
   :set-system set-system
   :get-system-ns get-system-ns
   :set-system-ns set-system-ns})

(defn create-state-tracker
  []
  (->StateTracker))

(ns cmr.graph.system.impl.management
  (:require
    [cmr.graph.system.impl.state :as state]
    [com.stuartsierra.component :as component]
    [taoensso.timbre :as log]))

(def valid-stop-transitions #{:started :running})
(def invalid-init-transitions #{:initialized :started :running})
(def invalid-deinit-transitions #{:started :running})
(def invalid-start-transitions #{:started :running})
(def invalid-stop-transitions #{:stopped})
(def invalid-startup-transitions #{:running})
(def invalid-shutdown-transitions #{:uninitialized :shutdown})

(defn- resolve-by-name
  [an-ns a-fun]
  (resolve (symbol (str an-ns "/" a-fun))))

(defn- call-by-name
  [an-ns a-fun & args]
  (apply (resolve-by-name an-ns a-fun) args))

(defrecord StateManager [state])

(defn init
  ([this]
    (init this :default))
  ([this mode]
    (if (contains? invalid-init-transitions (state/get-status (:state this)))
      (log/warn "System has aready been initialized.")
      (do
        (state/set-system (:state this)
                          (call-by-name (state/get-system-ns (:state this))
                                        "init"))
        (state/set-status (:state this) :initialized)))
    (state/get-status (:state this))))

(defn deinit
  [this]
  (if (contains? invalid-deinit-transitions (state/get-status (:state this)))
    (log/error "System is not stopped; please stop before deinitializing.")
    (do
      (state/set-system (:state this) nil)
      (state/set-status (:state this) :uninitialized)))
  (state/get-status (:state this)))

(defn start
  ([this]
    (start this :default))
  ([this mode]
    (when (nil? (state/get-status (:state this)))
      (init mode))
    (if (contains? invalid-start-transitions (state/get-status (:state this)))
      (log/warn "System has already been started.")
      (do
        (state/set-system (:state this)
                          (component/start (state/get-system (:state this))))
        (state/set-status (:state this) :started)))
    (state/get-status (:state this))))

(defn stop
  [this]
  (if (contains? invalid-stop-transitions (state/get-status (:state this)))
    (log/warn "System already stopped.")
    (do
      (state/set-system (:state this)
                        (component/stop (state/get-system (:state this))))
      (state/set-status (:state this) :stopped)))
  (state/get-status (:state this)))

(defn restart
  ([this]
    (restart this :default))
  ([this mode]
    (stop this)
    (start this mode)))

(defn startup
  ([this]
    (startup this :default))
  ([this mode]
    (if (contains? invalid-startup-transitions (state/get-status (:state this)))
      (log/warn "System is already running.")
      (do
        (when-not (contains? invalid-init-transitions
                             (state/get-status (:state this)))
          (init this mode))
        (when-not (contains? invalid-start-transitions
                            (state/get-status (:state this)))
          (start this mode))
        (state/set-status (:state this) :running)
        (state/get-status (:state this))))))

(defn shutdown
  [this]
  "Stop a running system and de-initialize it.
  This is essentially a convenience wrapper for `stop` + `deinit`."
  (if (contains? invalid-shutdown-transitions (state/get-status (:state this)))
    (log/warn "System is already shutdown.")
    (do
      (when-not (contains? invalid-stop-transitions
                           (state/get-status (:state this)))
        (stop this))
      (when-not (contains? invalid-deinit-transitions
                           (state/get-status (:state this)))
        (deinit this))
      (state/set-status (:state this) :shutdown)
      (state/get-status (:state this)))))

(def behaviour
  {:init init
   :deinit deinit
   :start start
   :stop stop
   :restart restart
   :startup startup
   :shutdown shutdown})

(defn create-state-manager
  []
  (->StateManager (state/create-state-tracker)))

(ns cmr.graph.system.core
  (:require
    [cmr.graph.system.impl.management :as management]
    [cmr.graph.system.impl.state :as state])
  (:import
    (cmr.graph.system.impl.management StateManager)
    (cmr.graph.system.impl.state StateTracker)))

(defprotocol StateTrackerAPI
  (get-state [this])
  (set-state [this new-state])
  (get-status [this])
  (set-status [this value])
  (get-system [this])
  (set-system [this value])
  (get-system-ns [this])
  (set-system-ns [this value]))

(extend StateTracker
        StateTrackerAPI
        state/behaviour)

(defprotocol StateManagementAPI
  (init [this] [this mode])
  (deinit [this])
  (start [this] [this mode])
  (stop [this])
  (restart [this] [this mode])
  (startup [this] [this mode])
  (shutdown [this]))

(extend StateManager
        StateManagementAPI
        management/behaviour)

(def create-state-manager #'management/create-state-manager)

(ns cmr.graph.components.neo4j
  (:require
   [clojurewerkz.neocons.rest :as nr]
   [cmr.graph.components.config :as config]
   [com.stuartsierra.component :as component]
   [taoensso.timbre :as log]))

(defn get-conn
  [system]
  (get-in system [:neo4j :conn]))

(defrecord Neo4j [conn])

(nr/connect "http://localhost:7474/db/data/")

(defn start
  [this]
  (log/info "Starting Neo4j component ...")
  (let [conn (nr/connect (format "http://%s:%s%s"
                                 (config/neo4j-host this)
                                 (config/neo4j-port this)
                                 (config/neo4j-db-path this)))]
    (log/debug "Started Neo4j component.")
    (assoc this :conn conn)))

(defn stop
  [this]
  (log/info "Stopping Neo4j component ...")
  (log/debug "Stopped Neo4j component.")
  (assoc this :conn nil))

(def lifecycle-behaviour
  {:start start
   :stop stop})

(extend Neo4j
  component/Lifecycle
  lifecycle-behaviour)

(defn create-component
  []
  (map->Neo4j {}))

(ns cmr.graph.components.core
  (:require
    [cmr.graph.components.config :as config]
    [cmr.graph.components.elastic :as elastic]
    [cmr.graph.components.httpd :as httpd]
    [cmr.graph.components.logging :as logging]
    [cmr.graph.components.neo4j :as neo4j]
    [com.stuartsierra.component :as component]))

(def cfg
  {:config (config/create-component)})

(def log
  {:logging (component/using
             (logging/create-component)
             [:config])})

(def neo4j
  {:neo4j (component/using
           (neo4j/create-component)
           [:config :logging])})

(def elastic
  {:elastic (component/using
             (elastic/create-component)
             [:config :logging])})

(def httpd
  {:httpd (component/using
           (httpd/create-component)
           [:config :logging :neo4j :elastic])})

(defn initialize-bare-bones
  []
  (component/map->SystemMap
    (merge cfg
           log
           neo4j)))

(defn initialize-with-web
  []
  (component/map->SystemMap
    (merge cfg
           log
           neo4j
           elastic
           httpd)))

(def init-lookup
  {:basic #'initialize-bare-bones
   :web #'initialize-with-web})

(defn init
  ([]
    (init :web))
  ([mode]
    ((mode init-lookup))))

(ns cmr.graph.components.logging
  (:require
    [clojusc.twig :as logger]
    [com.stuartsierra.component :as component]
    [cmr.graph.components.config :as config]
    [taoensso.timbre :as log]))

(defrecord Logging [])

(defn start
  [this]
  (log/info "Starting logging component ...")
  (let [log-level (config/log-level this)
        log-nss (config/log-nss this)]
    (log/debug "Setting up logging with level" log-level)
    (log/debug "Logging namespaces:" log-nss)
    (logger/set-level! log-nss log-level)
    (log/debug "Started logging component.")
    this))

(defn stop
  [this]
  (log/info "Stopping logging component ...")
  (log/debug "Stopped logging component.")
  this)

(def lifecycle-behaviour
  {:start start
   :stop stop})

(extend Logging
  component/Lifecycle
  lifecycle-behaviour)

(defn create-component
  []
  (map->Logging {}))

(ns cmr.graph.components.elastic
  (:require
   [clojurewerkz.elastisch.rest :as esr]
   [cmr.graph.components.config :as config]
   [com.stuartsierra.component :as component]
   [taoensso.timbre :as log]))

(defrecord Elastic [conn])

(defn start
  [this]
  (log/info "Starting Elasticsearch component ...")
  (let [conn-mgr (clj-http.conn-mgr/make-reusable-conn-manager
                  {:timeout (config/elastic-timeout this)})
        conn (esr/connect (format "http://%s:%s"
                                  (config/elastic-host this)
                                  (config/elastic-port this))
                          {:connection-manager conn-mgr})]
    (log/debug "Started Elasticsearch component.")
    (assoc this :conn conn)))

(defn stop
  [this]
  (log/info "Stopping Elasticsearch component ...")
  (log/debug "Stopped Elasticsearch component.")
  (assoc this :conn nil))

(def lifecycle-behaviour
  {:start start
   :stop stop})

(extend Elastic
  component/Lifecycle
  lifecycle-behaviour)

(defn create-component
  []
  (map->Elastic {}))

(ns cmr.graph.components.config
  (:require
   [cmr.graph.config :as config]
   [com.stuartsierra.component :as component]
   [taoensso.timbre :as log]))

(defn- get-cfg
  [system]
  (get-in system [:config :data]))

(defn elastic-host
  [system]
  (get-in (get-cfg system) [:elastic :host]))

(defn elastic-port
  [system]
  (get-in (get-cfg system) [:elastic :port]))

(defn elastic-timeout
  [system]
  (get-in (get-cfg system) [:elastic :timeout]))

(defn http-port
  [system]
  (get-in (get-cfg system) [:httpd :port]))

(defn http-docroot
  [system]
  (get-in (get-cfg system) [:httpd :docroot]))

(defn log-level
  [system]
  (get-in (get-cfg system) [:logging :level]))

(defn log-nss
  [system]
  (get-in (get-cfg system) [:logging :nss]))

(defn neo4j-host
  [system]
  (get-in (get-cfg system) [:neo4j :host]))

(defn neo4j-port
  [system]
  (get-in (get-cfg system) [:neo4j :port]))

(defn neo4j-db-path
  [system]
  (get-in (get-cfg system) [:neo4j :db-path]))

(defrecord Config [data])

(defn start
  [this]
  (log/info "Starting config component ...")
  (log/debug "Started config component.")
  (let [cfg (config/data)]
    (log/trace "Built configuration:" cfg)
    (assoc this :data cfg)))

(defn stop
  [this]
  (log/info "Stopping config component ...")
  (log/debug "Stopped config component.")
  (assoc this :data nil))

(def lifecycle-behaviour
  {:start start
   :stop stop})

(extend Config
  component/Lifecycle
  lifecycle-behaviour)

(defn create-component
  []
  (map->Config {}))

(ns cmr.graph.components.httpd
  (:require
    [com.stuartsierra.component :as component]
    [cmr.graph.components.config :as config]
    [cmr.graph.rest.app :as rest-api]
    [org.httpkit.server :as server]
    [taoensso.timbre :as log]))

(defrecord HTTPD [])

(defn start
  [this]
  (log/info "Starting httpd component ...")
  (let [port (config/http-port this)
        server (server/run-server (rest-api/app this) {:port port})]
    (log/debugf "HTTPD is listening on port %s" port)
    (log/debug "Started httpd component.")
    (assoc this :server server)))

(defn stop
  [this]
  (log/info "Stopping httpd component ...")
  (if-let [server (:server this)]
    (server))
  (assoc this :server nil))

(def lifecycle-behaviour
  {:start start
   :stop stop})

(extend HTTPD
  component/Lifecycle
  lifecycle-behaviour)

(defn create-component
  []
  (map->HTTPD {}))

(ns cmr.graph.health
  (:require
   [clj-http.client :as httpc]))

(defn http-ok?
  [url]
  (if (= 200 (:status (httpc/head url)))
    true
    false))

(defn has-data?
  [x]
  (if (nil? x)
    false
    true))

(defn config-ok?
  [component]
  (has-data? (:config component)))

(defn elastic-ok?
  [component]
  (http-ok? (get-in component [:elastic :conn :uri])))

(defn logging-ok?
  [component]
  (has-data? (:logging component)))

(defn neo4j-ok?
  [component]
  (http-ok? (get-in component [:neo4j :conn :endpoint :uri])))

(defn components-ok?
  [component]
  {:config {:ok? (config-ok? component)}
   :httpd {:ok? true}
   :elastic {:ok? (elastic-ok? component)}
   :logging {:ok? (logging-ok? component)}
   :neo4j {:ok? (neo4j-ok? component)}})

(ns cmr.graph.collections.core
  (:require
   [cheshire.core :as json]
   [clojurewerkz.neocons.rest.cypher :as cypher]
   [clojurewerkz.neocons.rest.nodes :as nodes]
   [clojurewerkz.neocons.rest.relationships :as relations]
   [cmr.graph.queries.neo4j.collections :as query]))

(defn reset
  [conn]
  (cypher/tquery conn query/reset))

(defn get-all
  [conn]
  (cypher/tquery conn query/get-all))

(defn delete-all
  [conn]
  (cypher/tquery conn query/delete-all))

(defn delete-all-cascade
  [conn]
  (cypher/tquery conn query/delete-all-cascade))

(defn batch-add
  [conn ^String json])

(defn add-collection
  [conn ^String json])

(defn get-collection
  [conn ^String concept-id])

(defn delete-collection
  [conn ^String concept-id])

(defn update-collection
  [conn ^String concept-id])

(defn get-collections-via-related-urls
  [conn ^String concept-id]
  (cypher/tquery conn (query/get-urls-by-concept-id concept-id)))

(defn get-concept-ids-by-urls
  [conn urls]
  (cypher/tquery conn (query/get-concept-ids-by-urls urls)))

(ns cmr.graph.data.statement
  (:require
   [clojure.string :as string]
   [digest :as digest]))

(defn- coll-concept-id-key
  [coll]
  (string/replace (first (:concept-id coll)) "-" ))

(defn- coll->provider-node
  [coll]
  (let [provider-id (first (:provider-id coll))]
    [(format "CREATE (%s:Provider {ShortName:'%s'})"
             provider-id provider-id)]))

(defn- coll->coll-stmts
  [coll]
  (let [{:keys [provider-id concept-id entry-id version-id]} coll
        provider-id (first provider-id)
        concept-id (first concept-id)
        entry-id (first entry-id)
        version-id (first version-id)]
    [(format "CREATE (%s:Collection {conceptId: '%s', entryId:'%s', version:'%s'})"
             (coll-concept-id-key coll) concept-id entry-id version-id)
     (format "CREATE (%s)-[:PROVIDES {type:['collection']}]->(%s)"
             provider-id (coll-concept-id-key coll))]))

(defn- url-type->stmt
  [url-type coll-key]
  (let [url-type-key (str "A" (digest/md5 url-type))]
  [(format "CREATE (%s:UrlType {Value:'%s'})"
           url-type-key url-type)
   (format "CREATE (%s)-[:LINKS {type:['collection']}]->(%s)"
           url-type-key coll-key)]))

(defn- url->stmt
  [url coll-key]
  (let [url-key (str "A" (digest/md5 url))]
    [(format "CREATE (%s:Url {name:'%s'})"
             url-key url)
     (format "CREATE (%s)-[:LINKS_TO]->(%s)"
             coll-key url-key)
     (format "CREATE (%s)-[:IS_IN]->(%s)"
             url-key coll-key)]))

(defn- coll->related-urls-stmts
  [coll]
  (let [{:keys [related-urls]} coll
        url-types (distinct (mapv :type related-urls))
        urls (distinct (mapv :url related-urls))
        coll-key (coll-concept-id-key coll)]
    (concat (mapcat #(url-type->stmt % coll-key) url-types)
            (mapcat #(url->stmt % coll-key) urls))))

(defn coll->create-stmts
  [coll]
  (concat
   (coll->provider-node coll)
   (coll->coll-stmts coll)
   (coll->related-urls-stmts coll)))

(defn neo4j-statements
  [colls]
  (string/join "\n" (distinct (mapcat coll->create-stmts colls))))

(ns cmr.graph.data.tags
  (:require
   [clojure.data.codec.base64 :as b64]
   [clojure.edn :as edn])
  (:import
   (java.util.zip GZIPInputStream)
   (java.io ByteArrayInputStream)))

(defn gzip-base64-tag->edn
  [^String input]
  (-> input
      .getBytes
      b64/decode
      ByteArrayInputStream.
      GZIPInputStream.
      slurp
      edn/read-string))

(comment
 (gzip-base64-tag->edn "H4sIAAAAAAAAAIWPMQvCMBCF/8qRScEGxM3tiAELtSltFXEpoQkhII0ktR1K/7spio4Od8O7997HTcR5Q1vtAh2NDYG2o22p8bJ73nWgD+8UgWnewF/fYJX20btXspdAcqyQ/M11sreDbqz6BRme+DXZAcvE+QCYx+GlqEQGBZZ1yjIO7IglspqX6Q3rVOSwYlggW8Nl+0WaiAzv3SzFSeu8/rxi3BDJQdJ4VTYs6vwC7Me2uQkBAAA="))

(ns cmr.graph.data.import
  (:require
   [cheshire.core :as json]
   [clojure.data.csv :as csv]
   [clojure.java.io :as io]
   [clojurewerkz.neocons.rest.cypher :as cypher]
   [cmr.graph.data.statement :as statement]
   [cmr.graph.data.tags :as tags]
   [digest :as digest]))

(def json-collections-filename
  "data/all_public_collections_from_es.json")

(def test-file
  "data/testfile.json")

(def collection-csv-file
  "data/collections.csv")

(def collection-url-csv-file
  "data/collection_and_urls.csv")

(def collection-data-center-csv-file
  "data/collection_and_data_centers.csv")

(def collection-tag-csv-file
  "data/collection_and_tags.csv")

(def url-fields
  [:type :url])

(def relevant-fields
  [:concept-id :provider-id :entry-id :related-urls :data-center :version-id :metadata-format
   :tags-gzip-b64])

(defn parse-url-into-nodes
  [url]
  (select-keys (json/parse-string url true) url-fields))

(defn- parse-tags
  [tags-gzip-b64]
  (when tags-gzip-b64
    (tags/gzip-base64-tag->edn tags-gzip-b64)))

(defn md5-leo
  [value]
  (str "A" (digest/md5 value)))

(defn prepare-collection-for-import
  [collection]
  (update (select-keys (:fields collection) relevant-fields)
          :related-urls
          (fn [urls]
            (mapv parse-url-into-nodes urls))))

(defn read-json-file
  [filename]
  (json/parse-string (slurp (io/resource filename)) true))

(def collection-columns
  ["MD5Leo" "ConceptId" "ProviderId" "VersionId" "MetadataFormat"])

(defn collection->row
  [collection]
  (let [{:keys [provider-id concept-id version-id metadata-format]} collection]
    [(md5-leo (first concept-id))
     (first concept-id)
     (first provider-id)
     (first version-id)
     (first metadata-format)]))

(defn write-collection-csv
  [collections output-filename]
  (with-open [csv-file (io/writer output-filename)]
    (csv/write-csv csv-file [collection-columns])
    (csv/write-csv csv-file (mapv collection->row collections))))

(defn construct-collection-url-row
  [collection url]
  [(md5-leo (first (:concept-id collection)))
   (:url url)
   (:type url)])

(defn construct-collection-data-center-row
  [collection data-center]
  [(md5-leo (first (:concept-id collection)))
   data-center])

(defn write-collection-url-relationship-csv
  [collections output-filename]
  (let [rows (doall
              (for [collection collections
                    url (:related-urls collection)]
                (construct-collection-url-row collection url)))]
    (with-open [csv-file (io/writer output-filename)]
      (csv/write-csv csv-file [["CollectionMD5Leo" "URL" "URLType"]])
      (csv/write-csv csv-file rows))))

(defn write-collection-data-center-relationship-csv
  [collections output-filename]
  (let [rows (doall
              (for [collection collections
                    data-center (:data-center collection)]
                (construct-collection-data-center-row collection data-center)))]
    (with-open [csv-file (io/writer output-filename)]
      (csv/write-csv csv-file [["CollectionMD5Leo" "DataCenter"]])
      (csv/write-csv csv-file rows))))

(defn- construct-collection-tag-row
  [collection tag]
  (let [[tag-key tag-association-data] tag]
    [(md5-leo (first (:concept-id collection)))
     tag-key]))

(defn write-collection-tags-relationship-csv
  [collections output-filename]
  (let [rows (doall
              (for [collection collections
                    tag (parse-tags (first (:tags-gzip-b64 collection)))]
                (construct-collection-tag-row collection tag)))]
    (with-open [csv-file (io/writer output-filename)]
      (csv/write-csv csv-file [["CollectionMD5Leo" "TagKey"]])
      (csv/write-csv csv-file rows))))

(def import-statements
  ["CREATE CONSTRAINT ON (url:Url) ASSERT url.name IS UNIQUE"
   "CREATE CONSTRAINT ON (urlType:UrlType) ASSERT urlType.name IS UNIQUE"
   "CREATE CONSTRAINT ON (coll:Collection) ASSERT coll.md5Leo IS UNIQUE"
   "CREATE CONSTRAINT ON (dataCenter:DataCenter) ASSERT dataCenter.name IS UNIQUE"
   "CREATE CONSTRAINT ON (tag:Tag) ASSERT tag.name IS UNIQUE"
   "LOAD CSV WITH HEADERS FROM \"https://raw.githubusercontent.com/cmr-exchange/cmr-graph/master/resources/data/collections.csv\" AS csvLine
      MERGE (format:MetadataFormat {name: csvLine.MetadataFormat})
      MERGE (version:Version {name: csvLine.VersionId})
      MERGE (provider:Provider {name: csvLine.ProviderId})
      CREATE (coll:Collection {md5Leo: csvLine.MD5Leo, conceptId: csvLine.ConceptId})
      CREATE (coll)-[:OWNED_BY]->(provider)
      CREATE (coll)-[:FORMATTED_IN]->(format)
      CREATE (coll)-[:VERSION_IS]->(version)"
   "USING PERIODIC COMMIT 500
      LOAD CSV WITH HEADERS FROM \"https://raw.githubusercontent.com/cmr-exchange/cmr-graph/master/resources/data/collection_and_urls.csv\" AS csvLine
      MATCH (coll:Collection { md5Leo: csvLine.CollectionMD5Leo})
      MERGE (url:Url { name: csvLine.URL})
      MERGE (urlType:UrlType { name: csvLine.URLType})
      CREATE (coll)-[:LINKS_TO]->(url)
      CREATE (url)-[:HAS_TYPE]->(urlType)"
   "USING PERIODIC COMMIT 500
      LOAD CSV WITH HEADERS FROM \"https://raw.githubusercontent.com/cmr-exchange/cmr-graph/master/resources/data/collection_and_data_centers.csv\" AS csvLine
      MATCH (coll:Collection { md5Leo: csvLine.CollectionMD5Leo})
      MERGE (dataCenter:DataCenter { name: csvLine.DataCenter})
      CREATE (coll)-[:AFFILIATED_WITH]->(dataCenter)"
   "USING PERIODIC COMMIT 500
      LOAD CSV WITH HEADERS FROM \"https://raw.githubusercontent.com/cmr-exchange/cmr-graph/master/resources/data/collection_and_tags.csv\" AS csvLine
      MATCH (coll:Collection { md5Leo: csvLine.CollectionMD5Leo})
      MERGE (tag:Tag { name: csvLine.TagKey})
      CREATE (coll)-[:TAGGED_WITH]->(tag)"])

(defn import-all-data
  [conn]
  (doseq [statement import-statements]
    (cypher/tquery conn statement)))

(comment
 (prepare-collection-for-import (first (:hits (:hits (read-json-file json-collections-filename)))))
 (mapv prepare-collection-for-import (:hits (:hits (read-json-file test-file))))
 (prepare-collection-for-import (first (:hits (:hits (read-json-file json-collections-filename)))))

 (write-collection-csv (mapv prepare-collection-for-import (:hits (:hits (read-json-file json-collections-filename))))
                       (str "resources/" collection-csv-file))

 (write-collection-url-relationship-csv (mapv prepare-collection-for-import (:hits (:hits (read-json-file json-collections-filename))))
                                        (str "resources/" collection-url-csv-file))

 (write-collection-data-center-relationship-csv (mapv prepare-collection-for-import (:hits (:hits (read-json-file json-collections-filename))))
                                                (str "resources/" collection-data-center-csv-file))

 (write-collection-tags-relationship-csv (mapv prepare-collection-for-import (:hits (:hits (read-json-file json-collections-filename))))
                                         (str "resources/" collection-tag-csv-file))

 (mapv prepare-collection-for-import (:hits (:hits (read-json-file test-file))))
 (println
  (statement/neo4j-statements (mapv prepare-collection-for-import (:hits (:hits (read-json-file test-file)))))))

(ns cmr.graph.rest.route
  (:require
   [cmr.graph.components.config :as config]
   [cmr.graph.components.neo4j :as neo4j]
   [cmr.graph.health :as health]
   [cmr.graph.rest.handler :as handler]
   [reitit.ring :as ring]
   [taoensso.timbre :as log]))

(defn collections
  [httpd-component]
  (let [conn (neo4j/get-conn httpd-component)]
    [["/collections"
      {:get (handler/get-collections conn)
       :delete (handler/delete-collections conn)
       :post (handler/add-collections conn)
       :options handler/ok}]
     ["/collections/import"
      {:post (handler/import-collection-data conn)
       :options handler/ok}]
     ["/collection"
      {:post (handler/add-collection conn)
       :options handler/ok}]
     ["/collection/:concept-id"
      {:get (handler/get-collection conn)
       :delete (handler/delete-collection conn)
       :put (handler/update-collection conn)
       :options handler/ok}]]))

(defn relationships
  [httpd-component]
  (let [conn (neo4j/get-conn httpd-component)]
    [["/relationships/related-urls/collections/:concept-id"
      {:get (handler/get-collections-via-related-urls conn)
       :options handler/ok}]]))

(defn movie-demo
  [httpd-component]
  (let [conn (neo4j/get-conn httpd-component)]
    [["/demo/movie/graph/:limit" {
      :get (handler/movie-demo-graph conn)
      :options handler/ok}]
     ["/demo/movie/search" {
      :get (handler/movie-demo-search conn)
      :options handler/ok}]
     ["/demo/movie/title/:title" {
      :get (handler/movie-demo-title conn)
      :options handler/ok}]]))

(defn static
  [httpd-component]
  (let [docroot (config/http-docroot httpd-component)]
    [["/static/*" {
      :get (handler/static-files docroot)}]]))

(defn admin
  [httpd-component]
  (let [conn (neo4j/get-conn httpd-component)]
    [["/health" {
                 :get (handler/health httpd-component)
                 :options handler/ok}]
     ["/reset" {
                :delete (handler/reset conn)
                :options handler/ok}]
     ["/reload" {:post (handler/reload conn)
                 :options handler/ok}]
     ["/ping" {
               :get handler/ping
               :post handler/ping
               :options handler/ok}]]))

(defn dangerous
  [httpd-component]
  (let [conn (neo4j/get-conn httpd-component)]
    [["/queries/cypher"
      {:get (handler/cypher-injection-get conn)
       :post (handler/cypher-injection-post conn)
       :options handler/ok}]]))

(ns cmr.graph.rest.middleware
  (:require
   [cmr.graph.rest.response :as response]
   [taoensso.timbre :as log]))

(defn wrap-cors
  [handler]
  (fn [request]
    (response/cors request (handler request))))

(ns cmr.graph.rest.app
  (:require
   [clojure.java.io :as io]
   [cmr.graph.rest.handler :as handler]
   [cmr.graph.rest.middleware :as middleware]
   [cmr.graph.rest.route :as route]
   [ring.middleware.defaults :as ring-defaults]
   [reitit.ring :as ring]
   [taoensso.timbre :as log]))

(defn rest-api-routes
  [httpd-component]
  (concat
   (route/collections httpd-component)
   (route/relationships httpd-component)
   (route/static httpd-component)
   (route/movie-demo httpd-component)
   (route/admin httpd-component)
   (route/dangerous httpd-component)))

(defn app
  [httpd-component]
  (-> httpd-component
      rest-api-routes
      ring/router
      (ring/ring-handler handler/fallback)
      (ring-defaults/wrap-defaults ring-defaults/api-defaults)
      (middleware/wrap-cors)))

(ns cmr.graph.rest.handler
  (:require
   [clojure.java.io :as io]
   [clojurewerkz.neocons.rest.cypher :as cypher]
   [cmr.graph.data.import :as data-import]
   [clojusc.twig :as twig]
   [cmr.graph.collections.core :as collections]
   [cmr.graph.demo.movie :as movie]
   [cmr.graph.health :as health]
   [cmr.graph.rest.response :as response]
   [ring.middleware.file :as file-middleware]
   [ring.util.codec :as codec]
   [taoensso.timbre :as log]))

(defn get-collections
  [conn]
  (fn [request]
    (->> conn
         (collections/get-all)
         (response/json request))))

(defn delete-collections
  [conn]
  (fn [request]
    (let [cascade? (get-in request [:params :cascade])]
      (if (= "true" cascade?)
        (->> conn
             (collections/delete-all-cascade)
             (response/json request))
        (->> conn
             (collections/delete-all)
             (response/json request))))))

(defn add-collections
  [conn]
  (fn [request]
    (->> request
         :body
         slurp
         ;(collections/batch-add conn)
         ((fn [_] {:error :not-implemented}))
         (response/json request))))

(defn add-collection
  [conn]
  (fn [request]
    (->> request
         :body
         slurp
         ;(collections/add-collection conn)
         ((fn [_] {:error :not-implemented}))
         (response/json request))))

(defn get-collection
  [conn]
  (fn [request]
    (->> [:path-params :concept-id]
         (get-in request)
         ;(collections/get-collection conn)
         ((fn [_] {:error :not-implemented}))
         (response/json request))))

(defn delete-collection
  [conn]
  (fn [request]
    (->> [:path-params :concept-id]
         (get-in request)
         ;(collections/delete-collection conn)
         ((fn [_] {:error :not-implemented}))
         (response/json request))))

(defn update-collection
  [conn]
  (fn [request]
    (->> [:path-params :concept-id]
         (get-in request)
         ;(collections/update-collection conn)
         ((fn [_] {:error :not-implemented}))
         (response/json request))))

(defn- get-related-urls
  [conn concept-id]
  (let [result (collections/get-collections-via-related-urls conn concept-id)]
    (map #(get % "u.name") result)))

(defn get-collections-via-related-urls
  [conn]
  (fn [request]
    (let [related-urls (get-related-urls
                        conn
                        (get-in request [:path-params :concept-id]))
          result (collections/get-concept-ids-by-urls conn related-urls)
          concept-ids (distinct (map #(get % "c.conceptId") result))]
      (response/json request concept-ids))))

(defn import-collection-data
  [conn]
  (fn [request]
    (data-import/import-all-data conn)
    (response/ok request)))

(defn movie-demo-graph
  [conn]
  (fn [request]
    (->> [:path-params :limit]
         (get-in request)
         Integer.
         (movie/get-graph conn)
         (response/json request))))

(defn movie-demo-search
  [conn]
  (fn [request]
    (->> [:params :q]
         (get-in request)
         (movie/search conn)
         (response/json request))))

(defn movie-demo-title
  [conn]
  (fn [request]
    (->> [:path-params :title]
         (get-in request)
         (codec/percent-decode)
         (movie/get-movie conn)
         (response/json request))))

(defn health
  [component]
  (fn [request]
    (->> component
         health/components-ok?
         (response/json request))))

(defn reset
  [conn]
  (fn [request]
    ;; delete things in small increments to avoid hanging the system
    (collections/delete-all-cascade conn)
    (collections/reset conn)
    (response/ok request)))

(defn reload
  [conn]
  (fn [request]
    ;; delete things in small increments to avoid hanging the system
    (collections/delete-all-cascade conn)
    (collections/reset conn)
    (data-import/import-all-data conn)
    (response/ok request)))

(def ping
  (fn [request]
    (response/json request {:result :pong})))

(defn cypher-injection-get
  [conn]
  (fn [request]
    (->> [:params :q]
         (get-in request)
         (codec/percent-decode)
         (cypher/tquery conn)
         (response/json request))))

(defn cypher-injection-post
  [conn]
  (fn [request]
    (->> request
         :body
         slurp
         (cypher/tquery conn)
         (response/json request))))

(def ok
  (fn [request]
    (response/ok request)))

(def fallback
  (fn [request]
    (response/not-found request)))

(defn static-files
  [docroot]
  (fn [request]
    (if-let [doc-resource (.getPath (io/resource docroot))]
      (file-middleware/file-request request doc-resource))))

(ns cmr.graph.rest.response
  (:require
   [cheshire.core :as json]
   [ring.util.http-response :as response]
   [taoensso.timbre :as log]))

(defn ok
  [_request & args]
  (response/ok args))

(defn json
  [_request data]
  (-> data
      json/generate-string
      response/ok
      (response/content-type "application/json")))

(defn text
  [_request data]
  (-> data
      response/ok
      (response/content-type "text/plain")))

(defn not-found
  [_request]
  (response/content-type
   (response/not-found "Not Found")
   "plain/text"))

(defn cors
  [request response]
  (case (:request-method request)
    :options (-> response
                 (response/content-type "text/plain; charset=utf-8")
                 (response/header "Access-Control-Allow-Origin" "*")
                 (response/header "Access-Control-Allow-Methods" "POST, PUT, GET, DELETE, OPTIONS")
                 (response/header "Access-Control-Allow-Headers" "Content-Type")
                 (response/header "Access-Control-Max-Age" "2592000"))
    (response/header response "Access-Control-Allow-Origin" "*")))

(ns cmr.graph.config
  (:require
   [clojure.edn :as edn]
   [clojure.java.io :as io]))

(def config-file "config/cmr-graph/config.edn")

(defn data
  ([]
    (data config-file))
  ([filename]
    (with-open [rdr (io/reader (io/resource filename))]
      (edn/read (new java.io.PushbackReader rdr)))))