the rebuild of feedxcavator

.gitignore

@@ -0,0 +1,17 @@
+/.idea
+/.nrepl-port
+*.bak
+*jar
+/lib
+/classes
+/native
+/.lein-failures
+/checkouts
+/.lein-deps-sum
+*.clj~
+/target
+/war/WEB-INF/appengine-generated
+/war/WEB-INF/lib
+/.local/credentials
+/.local/backup
+update.cmd

.local/credentials.url

@@ -0,0 +1,2 @@
+[InternetShortcut]
+URL=https://console.cloud.google.com/apis/credentials/serviceaccountkey

.local/start-local-server.clj

@@ -0,0 +1,6 @@
+;; execute in REPL:
+(load-file "src/main/clj/feedxcavator/app.clj")
+(eval '(do
+  (in-ns 'feedxcavator.app)
+  (appengine-magic.core/start feedxcavator-app)))
+;(appengine-magic.feedxcavator.app/stop)

.local/terminal-settings.txt

@@ -0,0 +1 @@
+LEIN_HOME=d:/sandbox/.lein;JAVA_OPTS=-Duser.home=d:\sandbox;LEIN_JVM_OPTS=-Duser.home=d:\sandbox;HOME=d:\sandbox;APPENGINE_SDK=D:\software\dev\appengine-sdk

deploy.py

@@ -0,0 +1,44 @@
+import os, sys, shutil, subprocess
+import xml.etree.ElementTree as ET
+
+WAR = "war"
+XMLNS = {"g": "http://appengine.google.com/ns/1.0"}
+GCLOUD = os.environ["CLOUD_SDK"] + "/google-cloud-sdk/bin/gcloud.cmd"
+APPENGINE_WEB = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                             WAR, "WEB-INF", "appengine-web.xml")
+APPENGINE_WEB_BAK = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                 WAR, "WEB-INF", "appengine-web.bak")
+
+
+gae_config = ET.parse(APPENGINE_WEB)
+root = gae_config.getroot()
+version = root.find("g:version", XMLNS).text
+app_id = root.find("g:application", XMLNS).text
+
+
+def deploy():
+    subprocess.Popen([shutil.which("gcloud"), "app", "deploy", APPENGINE_WEB,
+                      "--version=" + version, "--project=" + app_id, "--quiet"],
+                     stdout=sys.stdout, stderr=sys.stderr).communicate()
+
+
+deploy()
+
+file = open(APPENGINE_WEB, 'r', encoding="utf-8")
+xml = file.read()
+xml = xml.replace("<!-- 'default' service -->", "<!-- 'default' service --><!--")\
+    .replace("<!-- end of 'default' service -->", "--><!-- end of 'default' service -->")\
+    .replace("<!-- 'worker' service --><!--", "<!-- 'worker' service -->") \
+    .replace("--><!-- end of 'worker' service -->", "<!-- end of 'worker' service -->")
+file.close()
+
+shutil.move(APPENGINE_WEB, APPENGINE_WEB_BAK)
+file = open(APPENGINE_WEB, 'w', encoding="utf-8")
+file.write(xml)
+file.close()
+
+deploy()
+
+os.remove(APPENGINE_WEB)
+shutil.move(APPENGINE_WEB_BAK, APPENGINE_WEB)
+

project.clj

@@ -0,0 +1,49 @@
+(defproject feedxcavator "2.1.0-SNAPSHOT"
+  :description "A programmable RSS server"
+  :license {:name "Eclipse Public License"
+            :url "http://www.eclipse.org/legal/epl-v10.html"}
+  :dependencies [[org.clojure/clojure "1.10.1"]
+                 [org.clojure/tools.macro "0.1.5"]
+                 [org.clojure/data.json "0.2.6"]
+                 [org.clojure/data.xml "0.0.8"]
+                 [org.clojure/clojurescript "1.10.520"]
+                 [ring/ring-json "0.5.0"]
+                 [compojure "1.6.1"]
+                 [enlive "1.1.6"]
+                 [hiccup "1.0.5"]
+                 [crate "0.2.4"]
+                 [org.flatland/ordered "1.5.7"]
+                 [prismatic/dommy "1.1.0"]
+                 [cljs-ajax "0.8.0"]
+                 [io.forward/yaml "1.0.9"]
+                 [appengine-magic "0.5.1-SNAPSHOT"]
+                 [com.oscaro/clj-gcloud-storage "0.71-1.2"]
+                 [net.sourceforge.htmlunit/htmlunit "2.36.0"]
+                 ;[clj-http "3.10.0"]
+                 ]
+  :plugins [[appengine-magic "0.5.1-SNAPSHOT"]
+            [lein-cljsbuild "1.1.7"]
+            ]
+  :aot [feedxcavator.app_servlet
+        feedxcavator.app_context]
+  :javac-options ["-target" "1.8" "-source" "1.8"]
+  :source-paths ["src/main/clj"]
+  :java-source-paths ["src/main/java"] ; Java source is stored separately.
+  :test-paths ["test" "src/test/clj"]
+  :resource-paths ["src/main/resources"]
+  :appengine-app-versions {:myxcavator "private"}
+  :repl-options {:init (load-file ".local/start-local-server.clj")}
+  :cljsbuild {
+              :builds [{
+                        ; The path to the top-level ClojureScript source directory:
+                        :source-paths ["src/main/cljs"]
+                        ; The standard ClojureScript compiler options:
+                        ; (See the ClojureScript compiler documentation for details.)
+                        :compiler     {
+                                       :output-to   "war/js/main.js"
+                                       :optimizations :whitespace ;:advanced
+                                       :pretty-print  true ;false
+                                       :infer-externs true
+                                       :externs ["src/main/js/externs.js"
+                                                 "war/js/jstree/jstree.js"]
+                                       }}]})

src/main/clj/feedxcavator/app.clj

@@ -0,0 +1,95 @@
+(ns feedxcavator.app
+  (:require [feedxcavator.core :as core]
+            [feedxcavator.code :as code]
+            [feedxcavator.code-user :as code-user] ; required
+            [feedxcavator.backend :as backend]
+            [feedxcavator.websub :as websub]
+            [compojure.handler :as handler]
+            [appengine-magic.core :as ae]
+            [ring.middleware.json :as ring-json]
+            [ring.middleware.multipart-params.byte-array :as ring-byte-array])
+  (:use compojure.core))
+
+(def user-code-compiled (atom false))
+
+(defroutes feedxcavator-app-routes
+  (GET "/" [] (core/redirect-to "/console"))
+  (GET "/console" [] (backend/main-page))
+
+  (GET "/front/feed-url" [uuid] (backend/get-feed-url uuid))
+  (GET "/front/list-feeds" [] (backend/list-feeds))
+  (GET "/front/list-task-feeds" [task] (backend/list-task-feeds task))
+  (GET "/front/create-new-feed" [] (backend/create-new-feed))
+  (GET "/front/feed-definition" [uuid] (backend/get-feed-definition uuid))
+  (POST "/front/feed-definition" request (backend/save-feed-definition request))
+  (POST "/front/test-feed" request (backend/test-feed request))
+  (GET "/front/delete-feed" [uuid] (backend/delete-feed uuid))
+  (GET "/front/get-code" [type] (backend/get-code type))
+  (POST "/front/save-code" request (backend/save-code request))
+  (GET "/front/get-log-entries" [n from] (backend/get-log-entries n from))
+  (GET "/front/clear-log" [] (backend/clear-log))
+  (GET "/front/get-auth-token" [] (backend/get-auth-token))
+  (GET "/front/gen-auth-token" [] (backend/gen-auth-token))
+  (GET "/front/get-settings" [] (backend/get-settings))
+  (POST "/front/save-settings" request (backend/save-settings request))
+  (GET "/front/backup-database" [] (backend/backup-database))
+  (POST "/front/restore-database" request (backend/restore-database request))
+
+  (ANY "/backend/check-schedules" [] (code/check-schedules))
+  (ANY "/backend/service-task-front" [] (backend/service-task-front))
+  (ANY "/backend/service-task-background" [] (backend/service-task-background))
+  (GET "/backend/run/:task" [task] (code/queue-task task))
+  (POST "/backend/execute-task" request (code/execute-queued-task request))
+
+  (POST "/api/wordfilter/add-regex" request (backend/add-filter-regex request))
+  (POST "/api/wordfilter/remove-regex" request (backend/remove-filter-regex request))
+  (POST "/api/wordfilter/list-words" request (backend/list-word-filter-words request))
+  (POST "/api/wordfilter/list" request (backend/list-word-filter request))
+
+  (GET "/feed/:suffix" [suffix] (backend/deliver-feed suffix))
+  (ANY "/image/:name" [name] (backend/serve-image name))
+  (ANY "/handler/:handler" request (code/execute-handler request))
+  (ANY "/redirect/:random/:url" [random url] (backend/redirect url))
+  (ANY "/websub" request (websub/hub-action request))
+
+  (ANY "/_ah/mail/*" request (backend/receive-mail request))
+
+  (ANY "*" [] (core/page-not-found)))
+
+(defn context-binder [handler]
+  (fn [req]
+    (let [server-name (:server-name req)
+          is-worker-inst (.startsWith server-name core/worker-url-prefix)]
+      (binding [core/*servlet-context* (:servlet-context req)
+                core/*remote-addr* (:remote-addr req)
+                core/*worker-instance* is-worker-inst
+                core/*app-host* (str "https://"
+                                     (if is-worker-inst
+                                       (.substring server-name (inc (.indexOf server-name ".")))
+                                       server-name)
+                                     (let [port (:server-port req)]
+                                       (when (and port (not= port 80) (not= port 443))
+                                         (str ":" port))))
+
+                core/*current-logging-source* nil
+                core/*last-http-response* (atom nil)
+                core/*last-http-error-code* (atom nil)
+                core/*last-http-network-error* (atom nil)]
+
+        (when (not @user-code-compiled)
+          (try
+            (code/compile-user-code)
+            (reset! user-code-compiled true)
+            (catch Exception e (.printStackTrace e))))
+
+        (handler req)))))
+
+
+(def feedxcavator-app-handler
+  (ring-json/wrap-json-body
+    (handler/site
+      (context-binder feedxcavator-app-routes)
+      {:multipart {:store (ring-byte-array/byte-array-store)}})
+    {:keywords? true :bigdecimals? true}))
+
+(ae/def-appengine-app feedxcavator-app #'feedxcavator-app-handler)

src/main/clj/feedxcavator/app_context.clj

@@ -0,0 +1,7 @@
+(ns feedxcavator.app_context
+  (:gen-class :implements [javax.servlet.ServletContextListener])
+  (:import [javax.servlet ServletContextEvent]))
+
+(defn -contextInitialized [this ^ServletContextEvent contextEvent])
+
+(defn -contextDestroyed [this ^ServletContextEvent contextEvent])

src/main/clj/feedxcavator/app_servlet.clj

@@ -0,0 +1,7 @@
+(ns feedxcavator.app_servlet
+  (:gen-class :extends javax.servlet.http.HttpServlet)
+  (:use [appengine-magic.servlet :only [make-servlet-service-method]])
+  (:use feedxcavator.app))
+
+(defn -service [this request response]
+  ((make-servlet-service-method feedxcavator-app) this request response))