diff --git a/app/config/index.js b/app/config/index.js index b7f2939..987b0ad 100644 --- a/app/config/index.js +++ b/app/config/index.js @@ -3,8 +3,8 @@ var path = require('path'); var nconf = require('nconf'); -var DEFAULT_HOST = 'https://openspending.org'; -var DEFAULT_BASE_PATH = ''; +const DEFAULT_HOST = 'https://openspending.org'; +const DEFAULT_BASE_PATH = ''; nconf.file({ file: path.join(__dirname, '/../../settings.json') diff --git a/app/front/scripts/controllers/download-package.js b/app/front/scripts/controllers/download-package.js index 3b3c9d0..81b28c5 100644 --- a/app/front/scripts/controllers/download-package.js +++ b/app/front/scripts/controllers/download-package.js @@ -1,11 +1,13 @@ 'use strict'; +var osAdminService = require('../services/admin'); angular.module('Application') .controller('DownloadPackageController', [ '$scope', 'PackageService', 'DownloadPackageService', - 'Configuration', 'ApplicationLoader', 'LoginService', + 'Configuration', 'ApplicationLoader', 'LoginService', '$interval', '$http', '$window', function($scope, PackageService, DownloadPackageService, - Configuration, ApplicationLoader, LoginService) { + Configuration, ApplicationLoader, LoginService, $interval, $http, $window) { + ApplicationLoader.then(function() { $scope.fileName = Configuration.defaultPackageFileName; $scope.attributes = PackageService.getAttributes(); @@ -16,6 +18,85 @@ angular.module('Application') $scope.login = LoginService; $scope.publishDataPackage = DownloadPackageService.publishDataPackage; $scope.state = DownloadPackageService.getState(true); + $scope.packageOBEUStatus = null; + $scope.obeuUrl = 'http://apps.openbudgets.eu' }); + + + $scope.publishAndRunWebHooks = function () { + $scope.publishDataPackage().finally(function (res) { + runWebHooks($scope.fiscalDataPackage.name); + }); + }; + + + $scope.redirectToViewer = function() { + console.log('querying'); + var url = $scope.obeuUrl + '/search/package?q=' + encodeURIComponent($scope.fiscalDataPackage.name) + '&size=1'; + $http.get(url, {withCredentials: false}).then( + function (res) { + console.log('good'); + var packageId = ''; + if (res.length > 0) { + packageId = res[0].name; + } + $window.open($scope.obeuUrl + '/' + packageId, '_blank'); + }, + function (err) { + console.log(err); + $window.open($scope.obeuUrl, '_blank'); + } + ) + }; + + + function runWebHooks(packageId) { + $scope.packageOBEUStatus = 'processing'; + // query data packages to extract the user's "owner id" + osAdminService.getDataPackages(LoginService.authToken, LoginService.userId).then(function (packages) { + console.log(packages); + var ownerId = packages[0].owner; + var dataPackageId = ownerId + ":" + packageId; + + // set the data package status to "published" (a.k.a. public) + osAdminService.togglePackagePublicationStatus(LoginService.permissionToken, {id: dataPackageId}).then( + function (res) { + // run web hooks for the package + var dataPackage = _.find(packages, {id: dataPackageId}); + + if (dataPackage) { + var token = LoginService.permissionToken; + osAdminService.runWebHooks(token, dataPackage).then(function(res) { + var iri = JSON.parse(res.response).iri; + var executionId = iri.substr(1 + iri.lastIndexOf('/'), iri.length); + var executionOverviewUrl = + 'http://apps.openbudgets.eu/linkedpipes/test/resources/executions/' + executionId + '/overview'; + pollPipelineUntilReady(executionOverviewUrl); + }); + } + } + ); + }, + function (err) { + console.log(err); + }); + }; + + function pollPipelineUntilReady(executionOverviewUrl) { + stop = $interval(function () { + $http.get('https://crossorigin.me/' + executionOverviewUrl, {withCredentials: false}).then( + function (res) { + console.log(res); + if (res.data.status['@id'].includes('finished')) { + $interval.cancel(stop); + $scope.packageOBEUStatus = 'ready'; + } + }, + function (err) { + console.log(err); + } + ); + }, 3000); + } } ]); diff --git a/app/front/scripts/services/admin.js b/app/front/scripts/services/admin.js new file mode 100644 index 0000000..f979d64 --- /dev/null +++ b/app/front/scripts/services/admin.js @@ -0,0 +1,359 @@ +'use strict'; + +/* global window */ + +var _ = require('lodash'); +var url = require('url'); +var downloader = require('./downloader'); +var Promise = require('bluebird'); + +module.exports.defaultSettingsUrl = 'config.json'; +module.exports.conductorUrl = 'https://openspending.org'; +module.exports.publishUrl = module.exports.conductorUrl + '/package/publish'; +module.exports.searchUrl = module.exports.conductorUrl + '/search/package'; +module.exports.pollInterval = 1000; + +var RemoteProcessingStatus = { + 'queued': 'Waiting in queue for an available processor', + 'initializing': 'Getting ready to load the package', + 'loading-datapackage': 'Reading the Fiscal Data Package', + 'validating-datapackage': 'Validagin Data Package correctness', + 'loading-resource': 'Loading Resource data', + 'deleting-table': 'Clearing previous rows for this dataset from the database', + 'creating-table': 'Preparing space for rows in the database', + 'loading-data-ready': 'Starting to load rows to database', + 'loading-data': 'Loading data into the database', + 'creating-babbage-model': 'Converting the Data Package into an API model', + 'saving-metadata': 'Saving package metadata', + 'done': 'Done', + 'fail': 'Failed' +}; + +function getSettings(settingsUrl) { + return Promise.resolve(window.globalConfig || {}); +} + +function updateUserProfile(authToken, profileData) { + var url = module.exports.conductorUrl + '/user/update'; + + profileData = _.pick(profileData || {}, [ + 'username' + ]); + + var data = _.chain(profileData) + .map(function(value, key) { + return encodeURIComponent(key) + '=' + encodeURIComponent(value); + }) + .push('jwt=' + encodeURIComponent(authToken)) + .join('&') + .value(); + + var options = { + method: 'POST' + }; + return downloader.getJson(url + '?' + data, options, true) + .then(function(result) { + if (!result.success) { + throw new Error(result.error); + } + return profileData; + }); +} + +function getDataPackageMetadata(dataPackage) { + var originUrl = dataPackage.origin_url || dataPackage.__origin_url || [ + '//datastore.openspending.org', + dataPackage.package.owner, + dataPackage.package.name, + 'datapackage.json' + ].join('/'); + originUrl = originUrl.replace(/^http:/, 'https:'); + + var totalCountOfRecords = (function(dataPackage) { + var result = 0; + + _.each(dataPackage.resources, function(resource) { + var count = parseInt(resource.count_of_rows, 10) || 0; + if (count > 0) { + result += count; + } + }); + if (result == 0) { + var count = parseInt(dataPackage.count_of_rows, 10) || 0; + if (count > 0) { + result = count; + } + } + + return result; + })(dataPackage.package); + + var totalSizeOfResources = _.chain(dataPackage.package.resources) + .map(function(resource) { + var result = parseInt(resource.bytes, 10) || 0; + return result > 0 ? result : 0; + }) + .sum() + .value(); + + return { + id: dataPackage.id, + name: dataPackage.package.name, + title: dataPackage.package.title, + description: dataPackage.package.description, + owner: dataPackage.package.owner, + isPublished: !dataPackage.package.private, + last_update: dataPackage.last_update ? dataPackage.last_update * 1000 : 0, + totalCountOfResources: _.get(dataPackage, 'package.resources.length', 0), + totalCountOfRecords: totalCountOfRecords, + totalSizeOfResources: totalSizeOfResources, + loadingStatus: (function() { + // Old packages will have no `loaded`/`loading_*` properties; + // treat them as successfully loaded. + + var isLoaded = _.isUndefined(dataPackage.loaded) ? true : + !!dataPackage.loaded; + + var loadingStatus = isLoaded ? 'done' : + (dataPackage.loaging_status || 'queued'); + + var isFailed = loadingStatus == 'fail'; + + var result = { + loaded: isLoaded, + failed: isFailed, + status: loadingStatus, + message: RemoteProcessingStatus[loadingStatus] || + RemoteProcessingStatus.queued, + error: isFailed ? dataPackage.loading_error : null + }; + + // Show UI message for failed and in-progress packages + result.showMessage = !result.loaded; + + // Calculate count of rows (if available) + result.countOfRecords = totalCountOfRecords; + result.processedRecords = 0; + + return result; + })(), + author: _.chain(dataPackage.package.author) + .split(' ') + .dropRight(1) + .join(' ') + .value(), + url: originUrl, + resources: _.chain(dataPackage.package.resources) + .map(function(resource) { + var resourceUrl = null; + if (resource.url) { + resourceUrl = resource.url; + } + if (resource.path) { + resourceUrl = url.resolve(originUrl, resource.path); + } + + if (resourceUrl) { + return { + name: resource.name, + url: resourceUrl + }; + } + }) + .filter() + .value() + }; +} + +function getDataPackageLoadingStatus(dataPackage) { + var url = module.exports.conductorUrl + '/package/status' + + '?datapackage=' + encodeURIComponent(dataPackage.url); + + return fetch(url) + .then(function(response) { + if (response.status != 200) { + throw new Error('Failed to load data from ' + response.url); + } + return response.json(); + }) + .then(function(response) { + if (!_.isObject(response)) { + throw new Error('Response should be an object'); + } + var responseStatus = ('' + response.status).toLowerCase(); + if (responseStatus == 'fail') { + throw new Error(response.error); // Go to .catch() + } else { + var progress = parseInt(response.progress, 10) || 0; + if (progress < 0) { + progress = 0; + } + return { + status: responseStatus, + progress: progress + }; + } + }); +} + +function pollPackageStatus(dataPackage, dataPackageUpdatedCallback) { + if (_.isObject(dataPackage.loadingStatus)) { + var status = dataPackage.loadingStatus; + // If package was not loaded and there is no error - it's still loading + if (!status.loaded && !status.error) { + dataPackage.loadingStatus.showMessage = true; + dataPackage.loadingStatus.processedRecords = 0; + var poll = function() { + getDataPackageLoadingStatus(dataPackage) + .then(function(result) { + var loadingStatus = dataPackage.loadingStatus; + + loadingStatus.loaded = result.status == 'done'; + loadingStatus.failed = false; + loadingStatus.status = result.status; + loadingStatus.message = RemoteProcessingStatus[result.status]; + loadingStatus.error = null; + loadingStatus.processedRecords = result.progress; + + if (loadingStatus.processedRecords > loadingStatus.countOfRecords) { + if (loadingStatus.countOfRecords > 0) { + loadingStatus.processedRecords = loadingStatus.countOfRecords; + } + } + + if (_.isFunction(dataPackageUpdatedCallback)) { + dataPackageUpdatedCallback(dataPackage); + } + + if (result.status != 'done') { + setTimeout(poll, module.exports.pollInterval); + } else { + loadingStatus.processedRecords = loadingStatus.countOfRecords; + } + }) + .catch(function(error) { + var loadingStatus = dataPackage.loadingStatus; + + loadingStatus.loaded = false; + loadingStatus.failed = true; + loadingStatus.status = 'fail'; + loadingStatus.message = RemoteProcessingStatus.fail; + loadingStatus.error = error.message; + + if (_.isFunction(dataPackageUpdatedCallback)) { + dataPackageUpdatedCallback(dataPackage); + } + }); + }; + poll(); + } + } + return dataPackage; +} + +function getDataPackages(authToken, userid, dataPackageUpdatedCallback) { + var url = module.exports.searchUrl + '?size=10000'; + if (authToken) { + url += '&jwt=' + encodeURIComponent(authToken); + } + if (userid) { + url += '&package.owner=' + encodeURIComponent(JSON.stringify(userid)); + } + return downloader.getJson(url).then(function(packages) { + return _.chain(packages) + .map(getDataPackageMetadata) + .map(function(dataPackage) { + return pollPackageStatus(dataPackage, dataPackageUpdatedCallback); + }) + .sortBy(function(item) { + return item.title; + }) + .value(); + }); +} + +function togglePackagePublicationStatus(permissionToken, dataPackage) { + var url = module.exports.conductorUrl + '/package/publish'; + + var data = _.chain({ + jwt: permissionToken, + id: dataPackage.id, + publish: 'toggle' + }) + .map(function(value, key) { + return encodeURIComponent(key) + '=' + encodeURIComponent(value); + }) + .join('&') + .value(); + + var options = { + method: 'POST' + }; + return downloader.getJson(url + '?' + data, options, true) + .then(function(result) { + if (!result.success) { + throw new Error(result.error); + } + dataPackage.isPublished = !!result.published; + return dataPackage; + }); +} + +function deletePackage(permissionToken, dataPackage) { + var url = module.exports.conductorUrl + '/package/delete'; + + var data = _.chain({ + jwt: permissionToken, + id: dataPackage.id + }) + .map(function(value, key) { + return encodeURIComponent(key) + '=' + encodeURIComponent(value); + }) + .join('&') + .value(); + + var options = { + method: 'POST' + }; + return downloader.getJson(url + '?' + data, options, true) + .then(function(result) { + if (!result.success) { + throw new Error(result.error); + } + return dataPackage; + }); +} + + +function runWebHooks(permissionToken, dataPackage) { + var url = module.exports.conductorUrl + '/package/run-hooks'; + + var data = _.chain({ + jwt: permissionToken, + id: dataPackage.id, + pipeline: 'https://apps.openbudgets.eu/linkedpipes/execute/fdp2rdf' + }) + .map(function(value, key) { + return encodeURIComponent(key) + '=' + encodeURIComponent(value); + }) + .join('&') + .value(); + + var options = { + method: 'POST' + }; + return downloader.getJson(url + '?' + data, options, true) + .then(function(result) { + if (!result.success) { + throw new Error(result.error); + } + return result; + }); +} + +module.exports.getSettings = getSettings; +module.exports.updateUserProfile = updateUserProfile; +module.exports.getDataPackages = getDataPackages; +module.exports.togglePackagePublicationStatus = togglePackagePublicationStatus; +module.exports.deletePackage = deletePackage; +module.exports.runWebHooks = runWebHooks; diff --git a/app/front/scripts/services/download-package.js b/app/front/scripts/services/download-package.js index cc50390..ad66bbb 100644 --- a/app/front/scripts/services/download-package.js +++ b/app/front/scripts/services/download-package.js @@ -81,11 +81,12 @@ angular.module('Application') }; result.publishDataPackage = function() { - state.packagePublicUrl = null; - state.isUploading = true; - PackageService.publish().then(function(files) { - state.uploads = files; - files.$promise + return $q(function (resolve, reject) { + state.packagePublicUrl = null; + state.isUploading = true; + PackageService.publish().then(function(files) { + state.uploads = files; + files.$promise .then(function() { var packageName = PackageService.getAttributes().name; var owner = LoginService.userId; @@ -98,9 +99,11 @@ angular.module('Application') }) .finally(function() { state.isUploading = false; + resolve(true); }); + return state; + }); }); - return state; }; return result; diff --git a/app/front/scripts/services/downloader.js b/app/front/scripts/services/downloader.js new file mode 100644 index 0000000..0d69329 --- /dev/null +++ b/app/front/scripts/services/downloader.js @@ -0,0 +1,33 @@ +'use strict'; + +require('isomorphic-fetch'); +var Promise = require('bluebird'); + +var cache = {}; + +module.exports = { + get: function(url, options, bypassCache) { + if (bypassCache || !cache[url]) { + var requestPromise = fetch(url, options).then(function(response) { + if (response.status != 200) { + throw new Error('Failed loading data from ' + response.url); + } + return response.text(); + }); + + if (bypassCache) { + return requestPromise; + } + cache[url] = requestPromise; + } + return new Promise(function(resolve, reject) { + cache[url].then(resolve).catch(reject); + }); + }, + getJson: function(url, options, bypassCache) { + return this.get(url, options, bypassCache).then(JSON.parse); + }, + clearCache: function() { + cache = {}; + } +}; diff --git a/app/front/scripts/services/login.js b/app/front/scripts/services/login.js index 4f88f5e..3c383dc 100644 --- a/app/front/scripts/services/login.js +++ b/app/front/scripts/services/login.js @@ -53,6 +53,7 @@ angular.module('Application') check.then(function(response) { attempting = false; token = response.token; + that.authToken = token; that.isLoggedIn = true; that.name = response.profile.name; that.email = response.profile.email; diff --git a/app/views/partials/steps/download-package.html b/app/views/partials/steps/download-package.html index 6d52d02..b5d83f4 100644 --- a/app/views/partials/steps/download-package.html +++ b/app/views/partials/steps/download-package.html @@ -55,7 +55,7 @@

Column Mapping

+
+ + Your dataset is being submitted to OpenBudgets. Please wait, we will notify you right here when it is ready. +
+ +
+ + Your dataset is ready in the OpenBudgets Platform! You can go there now! +
+ - Explore and visualize your data now! + Visualize your data in OpenSpending + + + + + Search your datapackage in OpenBudgets - Manage your datasets