diff --git a/chronicle/raconteur/Dockerfile b/chronicle/raconteur/Dockerfile new file mode 100644 index 0000000000..2a036023f0 --- /dev/null +++ b/chronicle/raconteur/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:16.04 + +RUN apt-get update && apt-get install -y ca-certificates \ + && apt-get clean + +RUN mkdir -p /raconteur +WORKDIR /raconteur +COPY raconteur /raconteur + +CMD ["/raconteur/raconteur"] diff --git a/chronicle/raconteur/Makefile b/chronicle/raconteur/Makefile new file mode 100644 index 0000000000..31eff533e2 --- /dev/null +++ b/chronicle/raconteur/Makefile @@ -0,0 +1,29 @@ +include ../../makelib +header = $(call baseheader, $(1), raconteur) + +DOCKER_REPO ?= $(DOCKER_STAGE_REPO) +DOCKER_IMAGE ?= raconteur +DOCKER_TAG ?= master + +build: + $(call header, Building) + GOOS=linux CGO_ENABLED=0 go build -o raconteur ./*.go + +test: + $(call header, Testing) + true + +docker: + $(call header, Dockerizing) + docker build -t $(DOCKER_IMAGE) . + +docker-push: + $(call header, Registering) + docker tag $(DOCKER_IMAGE) $(DOCKER_REPO)/$(DOCKER_IMAGE):$(DOCKER_TAG) + docker push $(DOCKER_REPO)/$(DOCKER_IMAGE):$(DOCKER_TAG) + +clean: + $(call header, Cleaning) + rm -rf ./pkg raconteur + +.PHONY: build test docker docker-push diff --git a/chronicle/raconteur/README.md b/chronicle/raconteur/README.md new file mode 100644 index 0000000000..46043d0396 --- /dev/null +++ b/chronicle/raconteur/README.md @@ -0,0 +1,4 @@ +# raconteur + +Processes activities, pulls out object activity occurred on and executes javascript +registered for that object/object type and activity type diff --git a/chronicle/raconteur/consumer.go b/chronicle/raconteur/consumer.go new file mode 100644 index 0000000000..089971e928 --- /dev/null +++ b/chronicle/raconteur/consumer.go @@ -0,0 +1,57 @@ +package main + +import ( + "encoding/base64" + "errors" + "log" + "os/exec" + + "github.com/FoxComm/highlander/middlewarehouse/models/activities" + "github.com/FoxComm/metamorphosis" +) + +type EventConsumer struct { + ApiUrl string +} + +func NewEventConsumer(apiUrl string) (*EventConsumer, error) { + if apiUrl == "" { + return nil, errors.New("API_URL is required") + } + + return &EventConsumer{apiUrl}, nil +} + +func (c EventConsumer) processActivity(activity activities.ISiteActivity) error { + log.Printf("Processing %s: %s", activity.Type(), activity.Data()) + + scriptText := "console.log('ACTIVITY: ' + type); api.get('/v1/public/products/' + data.cart.lineItems.skus[0].productFormId).then(function (resp) { console.log('RESPONSE: ');console.log(resp);});" + + data := base64.StdEncoding.EncodeToString([]byte(activity.Data())) + script := base64.StdEncoding.EncodeToString([]byte(scriptText)) + + //Get from DB + jwt := "test-jwt" + stripeKey := "test-stripe" + + out, err := exec.Command("node", "--harmony", "executor/main.js", c.ApiUrl, stripeKey, jwt, activity.Type(), data, script).CombinedOutput() + + log.Printf("OUT: %s\n", out) + log.Printf("ERR: %v\n", err) + return nil +} + +func (c EventConsumer) Handler(message metamorphosis.AvroMessage) error { + activity, err := activities.NewActivityFromAvro(message) + if err != nil { + log.Fatalf("Unable to read activity with error %s", err.Error()) + return err + } + + log.Printf("Activity: %s", activity.Type()) + err = c.processActivity(activity) + if err != nil { + log.Fatalf("Unable to parse json with error %s", err.Error()) + } + return err +} diff --git a/chronicle/raconteur/executor/main.js b/chronicle/raconteur/executor/main.js new file mode 100644 index 0000000000..98e95c7b00 --- /dev/null +++ b/chronicle/raconteur/executor/main.js @@ -0,0 +1,61 @@ +const _ = require('lodash'); +const vm = require('vm'); +const fox = require('api-js'); +const superagent = require('superagent'); + + +function createApi(apiBaseUrl ,stripeKey, options = {}) { + + if (!apiBaseUrl) { + throw new Error('API_URL is not defined in process.env'); + } + + let agent = superagent; + + if (options.preserveCookies) { + agent = superagent.agent(); + } + + return new fox.default({ + api_url: apiBaseUrl + '/api', + stripe_key: stripeKey, + agent, + handleResponse: options.handleResponse, + }); +} + +//Parse args +let apiUrl = process.argv[2]; +let stripeKey = process.argv[3]; +let jwt = process.argv[4]; +let type = process.argv[5]; +let dataBase64 = process.argv[6]; +let scriptBase64 = process.argv[7]; + +//decode arguments +let data = new Buffer(dataBase64, 'base64').toString('ascii') +let jsonData = JSON.parse(data); + +let scriptText = new Buffer(scriptBase64, 'base64').toString('ascii') + +//setup execution context +let api = createApi(apiUrl, stripeKey); + +console.log("ACTIVITY: " + type); +console.log("DATA: " + data); +console.log("SCRIPT: " + scriptText); + +const sandbox = { + console: console, + api: api, + type: type, + data: jsonData +}; + +//run script +const script = new vm.Script(scriptText); +script.runInNewContext(sandbox, {displayErrors: true}); + + + + diff --git a/chronicle/raconteur/main.go b/chronicle/raconteur/main.go new file mode 100644 index 0000000000..11b7915e59 --- /dev/null +++ b/chronicle/raconteur/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "log" + "os" + + "github.com/FoxComm/highlander/middlewarehouse/consumers" + "github.com/FoxComm/metamorphosis" +) + +const ( + groupId = "raconteur-1" + clientId = "raconteur" +) + +var apiUrl = os.Getenv("API_URL") + +func main() { + config, err := consumers.MakeConsumerConfig() + if err != nil { + log.Fatalf("Unable to initialize consumer with error %s", err.Error()) + } + + consumer, err := metamorphosis.NewConsumer(config.ZookeeperURL, config.SchemaRepositoryURL, metamorphosis.OffsetResetLargest) + if err != nil { + log.Fatalf("Unable to connect to Kafka with error %s", err.Error()) + } + + consumer.SetGroupID(groupId) + consumer.SetClientID(clientId) + + oc, err := NewEventConsumer(apiUrl) + + consumer.RunTopic(config.Topic, oc.Handler) +} diff --git a/chronicle/raconteur/package.json b/chronicle/raconteur/package.json new file mode 100644 index 0000000000..36803efc4c --- /dev/null +++ b/chronicle/raconteur/package.json @@ -0,0 +1,22 @@ +{ + "name": "executor", + "version": "1.0.0", + "description": "", + "main": "main.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "api-js": "git+ssh://git@github.com/FoxComm/api-js.git", + "crypto-js": "^3.1.9-1", + "faker": "^3.1.0", + "lodash": "^4.17.4", + "nightmare": "2.7.0", + "stripe": "^4.15.0", + "superagent": "^3.4.0", + "system": "^1.2.0" + } +} diff --git a/phoenix-scala/app/failures/ObjectFailures.scala b/phoenix-scala/app/failures/ObjectFailures.scala index aefc0f7bd8..b401a4d201 100644 --- a/phoenix-scala/app/failures/ObjectFailures.scala +++ b/phoenix-scala/app/failures/ObjectFailures.scala @@ -9,6 +9,11 @@ object ObjectFailures { NotFoundFailure404(s"Context with name $name cannot be found") } + object ObjectNotFoundForContext { + def apply(id: Int, context: String): NotFoundFailure404 = + NotFoundFailure404(s"Object $id not found for context $context") + } + case class ObjectValidationFailure(kind: String, id: Int, errors: String) extends Failure { override def description = s"Object $kind with id=$id doesn't pass validation: $errors" } diff --git a/phoenix-scala/app/models/chronicle/ActivityKindHandlers.scala b/phoenix-scala/app/models/chronicle/ActivityKindHandlers.scala new file mode 100644 index 0000000000..d9cc6b5b11 --- /dev/null +++ b/phoenix-scala/app/models/chronicle/ActivityKindHandlers.scala @@ -0,0 +1,45 @@ +package models.chronicle + +import java.time.Instant + +import models.objects.{GenericObjects, GenericObject, ObjectHead, ObjectHeads} +import models.objects.ObjectUtils.InsertResult +import shapeless._ +import slick.lifted.Tag +import utils.Validation +import utils.db.ExPostgresDriver.api._ +import utils.db._ + +import com.github.tminglei.slickpg._ + +case class ActivityKindHandler(id: Int = 0, + scope: LTree, + kind: String, + activityHandlerHead: Int, + createdAt: Instant = Instant.now) + extends FoxModel[ActivityKindHandler] + with Validation[ActivityKindHandler] + +class ActivityKindHandlers(tag: Tag) extends FoxTable[ActivityKindHandler](tag, "users") { + def id = column[Int]("id", O.PrimaryKey, O.AutoInc) + def scope = column[LTree]("scope") + def kind = column[String]("kind") + def activityHandlerHead = column[Int]("activity_handler_head") + def createdAt = column[Instant]("created_at") + + def * = + (id, scope, kind, activityHandlerHead, createdAt) <> ((ActivityKindHandler.apply _).tupled, ActivityKindHandler.unapply) + + def activityHead = + foreignKey(GenericObjects.tableName, activityHandlerHead, GenericObjects)(_.id) +} + +object ActivityKindHandlers + extends FoxTableQuery[ActivityKindHandler, ActivityKindHandlers](new ActivityKindHandlers(_)) + with ReturningId[ActivityKindHandler, ActivityKindHandlers] { + + val returningLens: Lens[ActivityKindHandler, Int] = lens[ActivityKindHandler].id + + def filterByKind(kind: String): QuerySeq = + filter(_.kind === kind) +} diff --git a/phoenix-scala/app/models/objects/GenericObjects.scala b/phoenix-scala/app/models/objects/GenericObjects.scala new file mode 100644 index 0000000000..44c81d45c1 --- /dev/null +++ b/phoenix-scala/app/models/objects/GenericObjects.scala @@ -0,0 +1,60 @@ +package models.objects + +import java.time.Instant + +import shapeless._ +import slick.lifted.Tag +import utils.Validation +import utils.db.ExPostgresDriver.api._ +import utils.db._ + +import com.github.tminglei.slickpg._ + +case class GenericObject(id: Int = 0, + scope: LTree, + contextId: Int, + kind: String, + shadowId: Int, + formId: Int, + commitId: Int, + updatedAt: Instant = Instant.now, + createdAt: Instant = Instant.now, + archivedAt: Option[Instant] = None) + extends FoxModel[GenericObject] + with Validation[GenericObject] + with ObjectHead[GenericObject] { + + def withNewShadowAndCommit(shadowId: Int, commitId: Int): GenericObject = + this.copy(shadowId = shadowId, commitId = commitId) +} + +class GenericObjects(tag: Tag) extends ObjectHeads[GenericObject](tag, "generic_objects") { + + def kind = column[String]("kind") + + def * = + (id, scope, contextId, kind, shadowId, formId, commitId, updatedAt, createdAt, archivedAt) <> + ((GenericObject.apply _).tupled, GenericObject.unapply) +} + +object GenericObjects + extends FoxTableQuery[GenericObject, GenericObjects](new GenericObjects(_)) + with ReturningId[GenericObject, GenericObjects] { + + val returningLens: Lens[GenericObject, Int] = lens[GenericObject].id + + def filterByContext(contextId: Int): QuerySeq = + filter(_.contextId === contextId) + + def filterByFormId(formId: Int): QuerySeq = + filter(_.formId === formId) + + def withContextAndGenericObject(contextId: Int, genericObjectId: Int): QuerySeq = + filter(_.contextId === contextId).filter(_.formId === genericObjectId) + + def withContextAndForm(contextId: Int, formId: Int): QuerySeq = + filter(_.contextId === contextId).filter(_.formId === formId) + + def withContextAndKind(contextId: Int, kind: String): QuerySeq = + filter(_.contextId === contextId).filter(_.kind === kind) +} diff --git a/phoenix-scala/app/models/objects/IlluminatedObject.scala b/phoenix-scala/app/models/objects/IlluminatedObject.scala index 8607dfeede..bf33a691e7 100644 --- a/phoenix-scala/app/models/objects/IlluminatedObject.scala +++ b/phoenix-scala/app/models/objects/IlluminatedObject.scala @@ -10,12 +10,13 @@ case class IlluminatedContext(name: String, attributes: Json) * An IlluminatedObject is what you get when you combine the product shadow and * the product. */ -case class IlluminatedObject(id: Int = 0, attributes: Json) extends Identity[IlluminatedObject] +case class IlluminatedObject(id: Int = 0, kind: String, attributes: Json) + extends Identity[IlluminatedObject] object IlluminatedObject { def illuminate(form: ObjectForm, shadow: ObjectShadow): IlluminatedObject = { val attributes = IlluminateAlgorithm.projectAttributes(form.attributes, shadow.attributes) - IlluminatedObject(id = form.id, attributes = attributes) + IlluminatedObject(id = form.id, kind = form.kind, attributes = attributes) } } diff --git a/phoenix-scala/app/payloads/GenericObjectPayloads.scala b/phoenix-scala/app/payloads/GenericObjectPayloads.scala new file mode 100644 index 0000000000..0336f9f015 --- /dev/null +++ b/phoenix-scala/app/payloads/GenericObjectPayloads.scala @@ -0,0 +1,13 @@ +package payloads + +import utils.aliases._ + +object GenericObjectPayloads { + + case class CreateGenericObject(kind: String, + attributes: Map[String, Json], + schema: Option[String] = None, + scope: Option[String] = None) + + case class UpdateGenericObject(attributes: Map[String, Json]) +} diff --git a/phoenix-scala/app/responses/ObjectResponses.scala b/phoenix-scala/app/responses/ObjectResponses.scala index d66130ff97..73a36b030f 100644 --- a/phoenix-scala/app/responses/ObjectResponses.scala +++ b/phoenix-scala/app/responses/ObjectResponses.scala @@ -36,10 +36,10 @@ object ObjectResponses { object IlluminatedObjectResponse { - case class Root(id: Int, attributes: Json) + case class Root(id: Int, kind: String, attributes: Json) def build(s: IlluminatedObject): Root = - Root(id = s.id, attributes = s.attributes) + Root(id = s.id, kind = s.kind, attributes = s.attributes) } object ObjectSchemaResponse { diff --git a/phoenix-scala/app/routes/admin/ObjectRoutes.scala b/phoenix-scala/app/routes/admin/ObjectRoutes.scala index 0bbf6ad7ed..205d650df7 100644 --- a/phoenix-scala/app/routes/admin/ObjectRoutes.scala +++ b/phoenix-scala/app/routes/admin/ObjectRoutes.scala @@ -2,33 +2,57 @@ package routes.admin import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route -import utils.http.JsonSupport._ + import models.account.User +import payloads.GenericObjectPayloads._ import services.Authenticator.AuthData -import utils.aliases._ import services.objects.ObjectSchemasManager +import services.objects.ObjectManager +import utils.aliases._ import utils.http.CustomDirectives._ import utils.http.Http._ +import utils.http.JsonSupport._ object ObjectRoutes { def routes(implicit ec: EC, db: DB, auth: AuthData[User]): Route = { activityContext(auth) { implicit ac ⇒ - pathPrefix("object" / "schemas") { - (get & pathEnd) { - getOrFailures { - ObjectSchemasManager.getAllSchemas() - } - } ~ - pathPrefix("byKind") { - (get & path(Segment)) { kind ⇒ + pathPrefix("object") { + pathPrefix("schemas") { + (get & pathEnd) { getOrFailures { - ObjectSchemasManager.getSchemasForKind(kind) + ObjectSchemasManager.getAllSchemas() + } + } ~ + pathPrefix("byKind") { + (get & path(Segment)) { kind ⇒ + getOrFailures { + ObjectSchemasManager.getSchemasForKind(kind) + } + } + } ~ + (get & pathPrefix("byName") & path(Segment)) { schemaName ⇒ + getOrFailures { + ObjectSchemasManager.getSchema(schemaName) } } } ~ - (get & pathPrefix("byName") & path(Segment)) { schemaName ⇒ - getOrFailures { - ObjectSchemasManager.getSchema(schemaName) + pathPrefix(Segment) { (context) ⇒ + (post & pathEnd & entity(as[CreateGenericObject])) { payload ⇒ + mutateOrFailures { + ObjectManager.create(payload, context) + } + } ~ + pathPrefix(IntNumber) { id ⇒ + (get & pathEnd) { + getOrFailures { + ObjectManager.getIlluminated(id, context) + } + } + (patch & pathEnd & entity(as[UpdateGenericObject])) { payload ⇒ + mutateOrFailures { + ObjectManager.update(id, payload, context) + } + } } } } diff --git a/phoenix-scala/app/services/objects/ObjectManager.scala b/phoenix-scala/app/services/objects/ObjectManager.scala index d34fd8f94a..caa169a743 100644 --- a/phoenix-scala/app/services/objects/ObjectManager.scala +++ b/phoenix-scala/app/services/objects/ObjectManager.scala @@ -3,7 +3,9 @@ package services.objects import failures._ import failures.ObjectFailures._ import models.objects._ +import models.account.Scope import payloads.ContextPayloads._ +import payloads.GenericObjectPayloads._ import responses.ObjectResponses._ import utils.aliases._ import utils.db.ExPostgresDriver.api._ @@ -53,6 +55,110 @@ object ObjectManager { def mustFindShadowById404(id: Int)(implicit ec: EC): DbResultT[ObjectShadow] = ObjectShadows.findOneById(id).mustFindOr(NotFoundFailure404(ObjectShadow, id)) + def create(payload: CreateGenericObject, contextName: String)( + implicit ec: EC, + db: DB, + au: AU): DbResultT[IlluminatedObjectResponse.Root] = + for { + context ← * <~ ObjectContexts + .filterByName(contextName) + .mustFindOneOr(ObjectContextNotFound(contextName)) + genericObject ← * <~ createInternal(payload, context) + } yield + IlluminatedObjectResponse.build( + IlluminatedObject.illuminate(genericObject.form, genericObject.shadow)) + + case class CreateInternalResult(genericObject: GenericObject, + commit: ObjectCommit, + form: ObjectForm, + shadow: ObjectShadow) + extends FormAndShadow { + override def update(form: ObjectForm, shadow: ObjectShadow): FormAndShadow = + copy(form = form, shadow = shadow) + } + + def createInternal(payload: CreateGenericObject, context: ObjectContext)( + implicit ec: EC, + au: AU): DbResultT[CreateInternalResult] = { + val fs = FormAndShadow.fromPayload(payload.kind, payload.attributes) + for { + scope ← * <~ Scope.resolveOverride(payload.scope) + ins ← * <~ ObjectUtils.insert(fs.form, fs.shadow, payload.schema) + genericObject ← * <~ GenericObjects.create( + GenericObject(scope = scope, + kind = payload.kind, + contextId = context.id, + formId = ins.form.id, + shadowId = ins.shadow.id, + commitId = ins.commit.id)) + } yield CreateInternalResult(genericObject, ins.commit, ins.form, ins.shadow) + } + + def update(genericObjectId: Int, payload: UpdateGenericObject, contextName: String)( + implicit ec: EC, + db: DB): DbResultT[IlluminatedObjectResponse.Root] = + for { + context ← * <~ ObjectContexts + .filterByName(contextName) + .mustFindOneOr(ObjectContextNotFound(contextName)) + genericObject ← * <~ updateInternal(genericObjectId, payload.attributes, context) + } yield + IlluminatedObjectResponse.build( + IlluminatedObject.illuminate(genericObject.form, genericObject.shadow)) + + case class UpdateInternalResult(oldGenericObject: GenericObject, + genericObject: GenericObject, + form: ObjectForm, + shadow: ObjectShadow) + def updateInternal( + genericObjectId: Int, + attributes: Map[String, Json], + context: ObjectContext, + forceUpdate: Boolean = false)(implicit ec: EC, db: DB): DbResultT[UpdateInternalResult] = { + + for { + genericObject ← * <~ GenericObjects + .filter(_.contextId === context.id) + .filter(_.formId === genericObjectId) + .mustFindOneOr(ObjectNotFoundForContext(genericObjectId, context.name)) + fs = FormAndShadow.fromPayload(genericObject.kind, attributes) + update ← * <~ ObjectUtils.update(genericObject.formId, + genericObject.shadowId, + fs.form.attributes, + fs.shadow.attributes, + forceUpdate) + commit ← * <~ ObjectUtils.commit(update) + updated ← * <~ updateHead(genericObject, update.shadow, commit) + } yield UpdateInternalResult(genericObject, updated, update.form, update.shadow) + } + + def getIlluminated(id: Int, contextName: String)( + implicit ec: EC, + db: DB): DbResultT[IlluminatedObjectResponse.Root] = + for { + context ← * <~ ObjectContexts + .filterByName(contextName) + .mustFindOneOr(ObjectContextNotFound(contextName)) + genericObject ← * <~ GenericObjects + .filter(_.contextId === context.id) + .filter(_.formId === id) + .mustFindOneOr(NotFoundFailure404(GenericObject, id)) + form ← * <~ ObjectForms.mustFindById404(genericObject.formId) + shadow ← * <~ ObjectShadows.mustFindById404(genericObject.shadowId) + } yield IlluminatedObjectResponse.build(IlluminatedObject.illuminate(form, shadow)) + + private def updateHead( + genericObject: GenericObject, + shadow: ObjectShadow, + maybeCommit: Option[ObjectCommit])(implicit ec: EC): DbResultT[GenericObject] = + maybeCommit match { + case Some(commit) ⇒ + GenericObjects + .update(genericObject, genericObject.copy(shadowId = shadow.id, commitId = commit.id)) + case None ⇒ + DbResultT.good(genericObject) + } + def getFullObject[T <: ObjectHead[T]]( readHead: ⇒ DbResultT[T])(implicit ec: EC, db: DB): DbResultT[FullObject[T]] = for { diff --git a/phoenix-scala/sql/V4.090__activity_handlers.sql b/phoenix-scala/sql/V4.090__activity_handlers.sql new file mode 100644 index 0000000000..d92b0d4d51 --- /dev/null +++ b/phoenix-scala/sql/V4.090__activity_handlers.sql @@ -0,0 +1,38 @@ +create table generic_objects( + id serial primary key, + scope exts.ltree not null, + kind generic_string not null, + context_id integer not null references object_contexts(id) on update restrict on delete restrict, + shadow_id integer not null references object_shadows(id) on update restrict on delete restrict, + form_id integer not null references object_forms(id) on update restrict on delete restrict, + commit_id integer references object_commits(id) on update restrict on delete restrict, + updated_at generic_timestamp, + created_at generic_timestamp not null, + archived_at generic_timestamp +); + +create index generic_objects_object_context_idx on generic_objects (context_id); +create index generic_object_form_idx on generic_objects (form_id); + +--maps activity handlers to activities of a certain kind +create table activity_kind_handlers( + id serial primary key, + scope exts.ltree not null, + kind generic_string, + generic_object integer not null references generic_objects(id) on update restrict on delete restrict, + created_at generic_timestamp +); + +--maps activity handlers to activities of a certain kind and objectId/dimension +create table activity_kind_object_handlers( + id serial primary key, + scope exts.ltree not null, + kind generic_string, + dimension generic_string, + object_id generic_string, + generic_object integer not null references generic_objects(id) on update restrict on delete restrict, + created_at generic_timestamp +); + +create index activity_kind_handler_kindidx on activity_kind_handlers(scope, kind); +create index activity_kind_object_handler_kind_dim_obj_idx on activity_kind_object_handlers(scope, kind, dimension, object_id);