-
Notifications
You must be signed in to change notification settings - Fork 114
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1209 from snowplow/release/r133
R133
- Loading branch information
Showing
3 changed files
with
342 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
185 changes: 185 additions & 0 deletions
185
schemas/com.snowplowanalytics.monitoring.batch/load_succeeded/jsonschema/3-0-0
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,185 @@ | ||
| { | ||
| "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", | ||
| "description": "Schema for a data load succeeding", | ||
| "self": { | ||
| "vendor": "com.snowplowanalytics.monitoring.batch", | ||
| "name": "load_succeeded", | ||
| "format": "jsonschema", | ||
| "version": "3-0-0" | ||
| }, | ||
|
|
||
| "type": "object", | ||
| "properties": { | ||
| "shredding": { | ||
| "description": "Information about the batch taken from transformation step, isomorphic to shredding_complete schema", | ||
| "type": "object", | ||
| "properties": { | ||
| "base": { | ||
| "description": "Blob storage path to the root of the batch", | ||
| "type": "string", | ||
| "format": "uri", | ||
| "maxLength": 1024 | ||
| }, | ||
| "compression": { | ||
| "description": "File compression type", | ||
| "enum": ["GZIP", "NONE"] | ||
| }, | ||
| "typesInfo": { | ||
| "description": "Info about schemas used in events and output formats in the respective batch", | ||
| "type": "object", | ||
| "oneOf": [ | ||
| { | ||
| "properties": { | ||
| "transformation": { | ||
| "description": "Type of the transformation", | ||
| "enum": ["SHREDDED"] | ||
| }, | ||
| "types": { | ||
| "description": "Set of Iglu URIs and the format they were shredded into", | ||
| "type": "array", | ||
| "items": { | ||
| "properties": { | ||
| "schemaKey": { | ||
| "description": "Iglu URI", | ||
| "type": "string", | ||
| "maxLength": 256 | ||
| }, | ||
| "format": { | ||
| "description": "File format into which the entities were shredded", | ||
| "enum": ["TSV", "JSON"] | ||
| }, | ||
| "snowplowEntity": { | ||
| "description": "Type of the self-describing JSON in the event", | ||
| "enum": ["SELF_DESCRIBING_EVENT", "CONTEXT"] | ||
| } | ||
| }, | ||
| "required": ["schemaKey", "format", "snowplowEntity"], | ||
| "additionalProperties": false | ||
| } | ||
| } | ||
| }, | ||
| "required": ["transformation", "types"], | ||
| "additionalProperties": false | ||
| }, | ||
| { | ||
| "properties": { | ||
| "transformation": { | ||
| "description": "Type of the transformation", | ||
| "enum": ["WIDEROW"] | ||
| }, | ||
| "fileFormat": { | ||
| "description": "Output file format", | ||
| "enum": ["JSON", "PARQUET"] | ||
| }, | ||
| "types": { | ||
| "description": "Set of Iglu URIs and the format they were shredded into", | ||
| "type": "array", | ||
| "items": { | ||
| "properties": { | ||
| "schemaKey": { | ||
| "description": "Iglu URI", | ||
| "type": "string", | ||
| "maxLength": 256 | ||
| }, | ||
| "snowplowEntity": { | ||
| "description": "Type of the self-describing JSON in the event", | ||
| "enum": ["SELF_DESCRIBING_EVENT", "CONTEXT"] | ||
| } | ||
| }, | ||
| "required": ["schemaKey", "snowplowEntity"], | ||
| "additionalProperties": false | ||
| } | ||
| } | ||
| }, | ||
| "required": ["transformation", "fileFormat", "types"], | ||
| "additionalProperties": false | ||
| } | ||
| ] | ||
| }, | ||
| "timestamps": { | ||
| "description": "Set of timestamps associated with the batch", | ||
| "type": "object", | ||
| "properties": { | ||
| "jobStarted": { | ||
| "description": "Time when the batch started being shredded", | ||
| "type": "string", | ||
| "format": "date-time" | ||
| }, | ||
| "jobCompleted": { | ||
| "description": "Time when the batch shredding has been finished (and the message being prepared)", | ||
| "type": "string", | ||
| "format": "date-time" | ||
| }, | ||
| "min": { | ||
| "description": "The earliest collector_stamp available in the batch", | ||
| "type": ["string", "null"], | ||
| "format": "date-time" | ||
| }, | ||
| "max": { | ||
| "description": "The latest collector_tstamp available in the batch", | ||
| "type": ["string", "null"], | ||
| "format": "date-time" | ||
| } | ||
| }, | ||
| "required": ["jobStarted", "jobCompleted", "min", "max"] | ||
| }, | ||
| "processor": { | ||
| "description": "Identificator of a shredder sent the message", | ||
| "type": "object", | ||
| "properties": { | ||
| "artifact": { | ||
| "description": "Name of the artifact", | ||
| "type": "string", | ||
| "maxLength": 64 | ||
| }, | ||
| "version": { | ||
| "description": "Semantic Version of the artifact", | ||
| "type": "string", | ||
| "maxLength": 16 | ||
| } | ||
| }, | ||
| "required": ["artifact", "version"] | ||
| }, | ||
| "count": { | ||
| "description": "Count of events in the batch, null means the count could not be calculated", | ||
| "type": ["object", "null"], | ||
| "properties": { | ||
| "good": { | ||
| "description": "Amount of good events in the batch", | ||
| "type": "integer", | ||
| "minimum": 0 | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "required": ["base", "compression", "typesInfo", "timestamps", "processor"], | ||
| "additionalProperties": false | ||
| }, | ||
| "application": { | ||
| "description": "Loader's name and version", | ||
| "type": "string", | ||
| "maxLength": 128 | ||
| }, | ||
| "attempt": { | ||
| "description": "How many attempts were taken to load the batch", | ||
| "type": "integer", | ||
| "minimum": 0 | ||
| }, | ||
| "loadingStarted": { | ||
| "description": "Time when loader started loading (first attempt)", | ||
| "type": "string", | ||
| "format": "date-time" | ||
| }, | ||
| "loadingCompleted": { | ||
| "description": "Time when loader finished loading (should match the ingestion timestamp in manifest)", | ||
| "type": "string", | ||
| "format": "date-time" | ||
| }, | ||
| "tags": { | ||
| "description": "Set of key value pairs providing additional information", | ||
| "type": "object" | ||
| } | ||
| }, | ||
| "additionalProperties": false, | ||
| "required": ["shredding", "application", "attempt", "loadingStarted", "loadingCompleted", "tags"] | ||
| } |
152 changes: 152 additions & 0 deletions
152
schemas/com.snowplowanalytics.snowplow.storage/shredding_complete/jsonschema/2-0-0
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| { | ||
| "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", | ||
| "description": "Message with batch metadata, passed from Shredder to Loader", | ||
| "self": { | ||
| "vendor": "com.snowplowanalytics.snowplow.storage", | ||
| "name": "shredding_complete", | ||
| "format": "jsonschema", | ||
| "version": "2-0-0" | ||
| }, | ||
| "type": "object", | ||
| "properties": { | ||
| "base": { | ||
| "description": "Blob storage path to the root of the batch", | ||
| "type": "string", | ||
| "format": "uri", | ||
| "maxLength": 1024 | ||
| }, | ||
| "compression": { | ||
| "description": "File compression type", | ||
| "enum": ["GZIP", "NONE"] | ||
| }, | ||
| "typesInfo": { | ||
| "description": "Info about schemas used in events and output formats in the respective batch", | ||
| "type": "object", | ||
| "oneOf": [ | ||
| { | ||
| "properties": { | ||
| "transformation": { | ||
| "description": "Type of the transformation", | ||
| "enum": ["SHREDDED"] | ||
| }, | ||
| "types": { | ||
| "description": "Set of Iglu URIs and the format they were shredded into", | ||
| "type": "array", | ||
| "items": { | ||
| "properties": { | ||
| "schemaKey": { | ||
| "description": "Iglu URI", | ||
| "type": "string", | ||
| "maxLength": 256 | ||
| }, | ||
| "format": { | ||
| "description": "File format into which the entities were shredded", | ||
| "enum": ["TSV", "JSON"] | ||
| }, | ||
| "snowplowEntity": { | ||
| "description": "Type of the self-describing JSONs in the event", | ||
| "enum": ["SELF_DESCRIBING_EVENT", "CONTEXT"] | ||
| } | ||
| }, | ||
| "required": ["schemaKey", "format", "snowplowEntity"], | ||
| "additionalProperties": false | ||
| } | ||
| } | ||
| }, | ||
| "required": ["transformation", "types"], | ||
| "additionalProperties": false | ||
| }, | ||
| { | ||
| "properties": { | ||
| "transformation": { | ||
| "description": "Type of the transformation", | ||
| "enum": ["WIDEROW"] | ||
| }, | ||
| "fileFormat": { | ||
| "description": "Output file format", | ||
| "enum": ["JSON", "PARQUET"] | ||
| }, | ||
| "types": { | ||
| "description": "Set of Iglu URIs and the format they were shredded into", | ||
| "type": "array", | ||
| "items": { | ||
| "properties": { | ||
| "schemaKey": { | ||
| "description": "Iglu URI", | ||
| "type": "string", | ||
| "maxLength": 256 | ||
| }, | ||
| "snowplowEntity": { | ||
| "description": "Type of the self-describing JSONs in the event", | ||
| "enum": ["SELF_DESCRIBING_EVENT", "CONTEXT"] | ||
| } | ||
| }, | ||
| "required": ["schemaKey", "snowplowEntity"], | ||
| "additionalProperties": false | ||
| } | ||
| } | ||
| }, | ||
| "required": ["transformation", "fileFormat", "types"], | ||
| "additionalProperties": false | ||
| } | ||
| ] | ||
| }, | ||
| "timestamps": { | ||
| "description": "Set of timestampts associated with the batch", | ||
| "type": "object", | ||
| "properties": { | ||
| "jobStarted": { | ||
| "description": "Time when the batch started being shredded", | ||
| "type": "string", | ||
| "format": "date-time" | ||
| }, | ||
| "jobCompleted": { | ||
| "description": "Time when the batch shredding has been finished (and the message being prepared)", | ||
| "type": "string", | ||
| "format": "date-time" | ||
| }, | ||
| "min": { | ||
| "description": "The earliest collector_stamp available in the batch", | ||
| "type": ["string", "null"], | ||
| "format": "date-time" | ||
| }, | ||
| "max": { | ||
| "description": "The latest collector_tstamp available in the batch", | ||
| "type": ["string", "null"], | ||
| "format": "date-time" | ||
| } | ||
| }, | ||
| "required": ["jobStarted", "jobCompleted", "min", "max"] | ||
| }, | ||
| "processor": { | ||
| "description": "Identificator of a shredder sent the message", | ||
| "type": "object", | ||
| "properties": { | ||
| "artifact": { | ||
| "description": "Name of the artifact", | ||
| "type": "string", | ||
| "maxLength": 64 | ||
| }, | ||
| "version": { | ||
| "description": "Semantic Version of the artifact", | ||
| "type": "string", | ||
| "maxLength": 16 | ||
| } | ||
| }, | ||
| "required": ["artifact", "version"] | ||
| }, | ||
| "count": { | ||
| "description": "Count of events in the batch, null means the count could not be calculated", | ||
| "type": ["object", "null"], | ||
| "properties": { | ||
| "good": { | ||
| "description": "Amount of good events in the batch", | ||
| "type": "integer", | ||
| "minimum": 0 | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "required": ["base", "compression", "typesInfo", "timestamps", "processor"], | ||
| "additionalProperties": false | ||
| } |