diff --git a/.travis.yml b/.travis.yml index 9f54f48..66e455e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: false language: python cache: pip -python: 3.5 +python: 3.6 before_install: - source .travis/before_install.sh diff --git a/.travis/install.sh b/.travis/install.sh index 912db12..d6e7ae9 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -19,5 +19,6 @@ if [[ $TRAVIS_BUILD_STAGE_NAME == 'Deploy' ]]; then fi pip install --upgrade pip +pip install docutils==0.17.1 pip install . pip install -r requirements-test.txt diff --git a/.zenodo.json b/.zenodo.json index 06bf478..8b38df3 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -31,5 +31,5 @@ }, "publication_date": "2017-11-22", "title": "grlc", - "version": "1.3.5" + "version": "1.3.6" } diff --git a/CITATION.cff b/CITATION.cff index 8521969..cb7c2ec 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -32,4 +32,4 @@ keywords: - "linked-data" - "semantic-web" - "linked-data-api" -version: "1.3.5" +version: "1.3.6" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 220827f..c8ee408 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,11 @@ Thank you very much for your interest in contributing to grlc! It's people like you that truly make the Semantic Web more accessible to everyone :) +## Communication channels + +If you would like to get in touch with the grlc developers, and with other users of grlc, you can reach us in two ways: + - Via Twitter, by using the grlc handle (**@grlcldapi**). Follow this account to hear about updates. + - Via the grlc [mailing list](https://groups.google.com/g/grlc-list/). Sign up to the mailing list to ask questions and make suggestions. + ## Filing bug reports The official channel to file bug reports is via our GitHub's [issue tracker](https://github.com/CLARIAH/grlc/issues). When doing so make sure that: @@ -17,6 +23,14 @@ As with bug reports, for requesting features please use the [issue tracker](http - Name the file/module if known/available - Tag the issue as **enhancement** +## Sending pull requests + +If you would like to contribute to the code directly, please send in a [pull request (PR)](https://github.com/CLARIAH/grlc/pulls). Please make sure that: + - The title of your PR briefly describes the content + - Describe in detail what your PR contributes + - If your PR addresses a specific issue, indicate the issue number + - Assign @albertmeronyo or @c-martinez as reviewer of your PR. + ## Testing environment To get started with hacking grlc, follow these steps to create a local testing environment (you'll need [docker](https://www.docker.com/) and [docker-compose](https://docs.docker.com/compose/)): @@ -46,6 +60,14 @@ services: You're good to pick any issue at the [issue tracker](https://github.com/CLARIAH/grlc/issues) marked as **enhancement** and start implementing it :) +## Governance model + +As creators of grlc, [@albertmeronyo](https://github.com/albertmeronyo) and [@c-martinez](http://github.com/c-martinez) are benevolent dictators for this project. This means that they have a final say of the direction of the project. This DOES NOT mean they are not willing to listen to suggestion (on the contrary, they *love* to hear new ideas)! + +## Contributing + +All grlc contributors will be listed in the [CONTRIBUTORS.md](CONTRIBUTORS.md) file. Also, [notes of new releases](https://github.com/CLARIAH/grlc/releases) will mention who contributed to that specific release. + ## Questions Please open an issue at the [issue tracker](https://github.com/CLARIAH/grlc/issues) and tag it as **question** diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md new file mode 100644 index 0000000..6622b7d --- /dev/null +++ b/CONTRIBUTORS.md @@ -0,0 +1,14 @@ +# Contributors +This is a list of all people who have contributed to grlc. Big thanks to everyone. + +[RinkeHoekstra](https://github.com/RinkeHoekstra) +[pasqLisena](https://github.com/pasqLisena) +[rlzijdeman](https://github.com/rlzijdeman) +[RoderickvanderWeerdt](https://github.com/RoderickvanderWeerdt) +[arnikz](https://github.com/arnikz) +[jetschni](https://github.com/jetschni) +[mwigham](https://github.com/mwigham) +[steltenpower](https://github.com/steltenpower) +[jspaaks](https://github.com/jspaaks) +[ecow](https://github.com/ecow) +[rapw3k](https://github.com/rapw3k) diff --git a/README.md b/README.md index a104ee8..25651bc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

-[![Join the chat at https://gitter.im/grlc](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/grlc/Lobby#) +[![PyPI version](https://badge.fury.io/py/grlc.svg)](https://badge.fury.io/py/grlc) [![DOI](https://zenodo.org/badge/46131212.svg)](https://zenodo.org/badge/latestdoi/46131212) [![Build Status](https://travis-ci.org/CLARIAH/grlc.svg?branch=master)](https://travis-ci.org/CLARIAH/grlc) @@ -103,7 +103,7 @@ queries: The API paths of all location types point to the generated [swagger-ui](https://swagger.io/) style API documentation. On the API documentation page, you can explore available API calls and execute individual API calls. -You can also view the swagger spec of your API, by visiting `/spec/`, for example: `http://grlc.io/api-git/CLARIAH/grlc-queries/spec/` +You can also view the swagger spec of your API, by visiting `/swagger`, for example: `http://grlc.io/api-git/CLARIAH/grlc-queries/swagger` ### grlc query execution When you call an API endpoint, grlc executes the SPARQL query for that endpoint by combining supplied parameters and decorators. @@ -192,6 +192,17 @@ Syntax: Example [query](https://github.com/CLARIAH/grlc-queries/blob/master/tags.rq) and the equivalent [API operation](http://grlc.io/api-git/CLARIAH/grlc-queries/#/group1/get_tags). +### `defaults` +Set the default value in the swagger-ui for a specific parameter in the query. + +Syntax: +``` +#+ defaults: +#+ - param_name: default_value +``` + +Example [query](https://github.com/CLARIAH/grlc-queries/blob/master/defaults.rq) and the equivalent [API operation](http://grlc.io/api-git/CLARIAH/grlc-queries/#/default/get_defaults). + ### `enumerate` Indicates which parameters of your query/operation should get enumerations (and get dropdown menus in the swagger-ui) using the given values from the SPARQL endpoint. The values for each enumeration variable can also be specified into the query decorators to save endpoint requests and speed up the API generation. @@ -218,7 +229,7 @@ Syntax: Example [query](https://github.com/CLARIAH/grlc-queries/blob/master/endpoint_url.rq) and the equivalent [API operation](http://grlc.io/api-git/CLARIAH/grlc-queries/#/default/get_endpoint_url). ### `transform` -Allows query results to be converted to the specified JSON structure, by using [SPARQLTransformer](https://github.com/D2KLab/py-sparql-transformer) syntax. +Allows query results to be converted to the specified JSON structure, by using [SPARQLTransformer](https://github.com/D2KLab/py-sparql-transformer) syntax. Notice that the response content type must be set to `application/json` for the transformation to take effect. Syntax: ``` @@ -351,7 +362,7 @@ Check our [contributing](CONTRIBUTING.md) guidelines for these and more, and joi If you cannot code, that's no problem! There's still plenty you can contribute: -- Share your experience at using grlc in Twitter (mention the handler **@grlcldapi**) +- Share your experience at using grlc in Twitter (mention the handle **@grlcldapi**) - If you are good with HTML/CSS, [let us know](mailto:albert.meronyo@gmail.com) ## Related tools diff --git a/requirements.txt b/requirements.txt index 2bbdc91..e7fa6d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ docopt==0.6.2 +docutils==0.17.1 Flask==1.0.2 Flask-Cors==3.0.6 gevent==1.4.0 @@ -10,14 +11,14 @@ keepalive==0.5 MarkupSafe==0.23 pyaml==18.11.0 pyparsing==2.0.7 -PyYAML==4.2b1 +PyYAML==5.4 rdflib==4.2.2 rdflib-jsonld==0.4.0 requests==2.20.0 six==1.12.0 simplejson==3.16.0 setuptools>=38.6.0 -SPARQLTransformer==1.9.0 +SPARQLTransformer==2.1.1 SPARQLWrapper==1.8.2 werkzeug>=0.16.0 PyGithub==1.43.5 diff --git a/src/fileLoaders.py b/src/fileLoaders.py index 8b1ccdb..efde936 100644 --- a/src/fileLoaders.py +++ b/src/fileLoaders.py @@ -340,7 +340,7 @@ def getRepoURI(self): def getLicenceURL(self): """Returns the URL of the license file in the specification.""" - return self.spec['licence'] if self.spec['licence'] else '' + return self.spec['licence'] if 'licence' in self.spec else None def getEndpointText(self): """Return content of endpoint file (endpoint.txt)""" diff --git a/src/server.py b/src/server.py index 10642bb..63347ab 100644 --- a/src/server.py +++ b/src/server.py @@ -81,18 +81,18 @@ def api_docs_local(): # Spec generation, JSON @app.route('/api-local/swagger', methods=['GET']) @app.route('/api/local/local/swagger', methods=['GET'], strict_slashes=False) # backward compatibility route -@app.route('/api-local/spec', methods=['GET']) # backward compatibility route -@app.route('/api/local/local/spec', methods=['GET'], strict_slashes=False) # backward compatibility route def swagger_spec_local(): """Swagger spec for local routes.""" return swagger_spec(user=None, repo=None, sha=None, content=None) # Callname execution @app.route('/api-local/', methods=['GET', 'POST']) +@app.route('/api-local/.', methods=['GET', 'POST']) @app.route('/api/local/local/', methods=['GET', 'POST'], strict_slashes=False) # backward compatibility route -def query_local(query_name): +@app.route('/api/local/local/.', methods=['GET', 'POST'], strict_slashes=False) # backward compatibility route +def query_local(query_name, content=None): """SPARQL query execution for local routes.""" - return query(user=None, repo=None, query_name=query_name) + return query(user=None, repo=None, query_name=query_name, content=content) ################################ ### Routes for URL HTTP APIs ### @@ -109,7 +109,6 @@ def api_docs_param(): # Spec generation, JSON @app.route('/api-url/swagger', methods=['GET']) -@app.route('/api-url/spec', methods=['GET']) # backward compatibility route def swagger_spec_param(): """Swagger spec for specifications loaded via http.""" spec_url = request.args['specUrl'] @@ -118,11 +117,12 @@ def swagger_spec_param(): # Callname execution @app.route('/api-url/', methods=['GET', 'POST']) -def query_param(query_name): +@app.route('/api-url/.', methods=['GET', 'POST']) +def query_param(query_name, content=None): """SPARQL query execution for specifications loaded via http.""" spec_url = request.args['specUrl'] glogger.debug("Spec URL: {}".format(spec_url)) - return query(user=None, repo=None, query_name=query_name, spec_url=spec_url) + return query(user=None, repo=None, query_name=query_name, spec_url=spec_url, content=content) ############################## ### Routes for GitHub APIs ### @@ -157,18 +157,8 @@ def api_docs_git(user, repo, subdir=None, sha=None): @app.route('/api////swagger', methods=['GET']) # backward compatibility route @app.route('/api///commit//swagger') # backward compatibility route @app.route('/api////commit//swagger') # backward compatibility route -@app.route('/api-git///spec', methods=['GET']) # backward compatibility route -@app.route('/api-git///swagger', methods=['GET']) # backward compatibility route -@app.route('/api-git///subdir//spec', methods=['GET']) # backward compatibility route @app.route('/api-git////swagger', methods=['GET']) # backward compatibility route -@app.route('/api-git///commit//spec') # backward compatibility route -@app.route('/api-git///subdir//commit//spec') # backward compatibility route -@app.route('/api-git////commit//spec') # backward compatibility route @app.route('/api-git////commit//swagger') # backward compatibility route -@app.route('/api///spec', methods=['GET']) # backward compatibility route -@app.route('/api////spec', methods=['GET']) # backward compatibility route -@app.route('/api///commit//spec') # backward compatibility route -@app.route('/api////commit//spec') # backward compatibility route def swagger_spec_git(user, repo, subdir=None, sha=None): """Swagger spec for specifications loaded from a Github repo.""" return swagger_spec(user, repo, subdir=subdir, spec_url=None, sha=sha, content=None) diff --git a/src/swagger.py b/src/swagger.py index b785cc0..243659d 100644 --- a/src/swagger.py +++ b/src/swagger.py @@ -29,7 +29,7 @@ def get_repo_info(loader, sha, prov_g): contact_name = loader.getContactName() contact_url = loader.getContactUrl() commit_list = loader.getCommitList() - licence_url = loader.getLicenceURL() + licence_url = loader.getLicenceURL() # This will be None if there is no license # Add the API URI as a used entity by the activity if prov_g: @@ -50,12 +50,13 @@ def get_repo_info(loader, sha, prov_g): 'contact': { 'name': contact_name, 'url': contact_url - }, - 'license': { + } + } + if licence_url: + info['license'] = { 'name': 'License', 'url': licence_url } - } if type(loader) is GithubLoader: basePath = '/api-git/' + user_repo + '/' @@ -82,7 +83,7 @@ def get_path_for_item(item): query = "\n" + json.dumps(query, indent=2) + "\n" description = item['description'] - description += '\n\n```{}```'.format(query) + description += '\n\n```\n{}\n```'.format(query) description += '\n\nSPARQL transformation:\n```json\n{}```'.format( item['transform']) if 'transform' in item else '' diff --git a/src/utils.py b/src/utils.py index ea936fa..15fc21a 100644 --- a/src/utils.py +++ b/src/utils.py @@ -124,7 +124,11 @@ def dispatchSPARQLQuery(raw_sparql_query, loader, requestArgs, acceptHeader, con glogger.debug("Sending query to SPARQL endpoint: {}".format(endpoint)) glogger.debug("=====================================================") - query_metadata = gquery.get_metadata(raw_sparql_query, endpoint) + try: + query_metadata = gquery.get_metadata(raw_sparql_query, endpoint) + except Exception as e: + # extracting metadata + return { 'error': str(e) }, 400, {} acceptHeader = 'application/json' if isinstance(raw_sparql_query, dict) else acceptHeader pagination = query_metadata['pagination'] if 'pagination' in query_metadata else "" @@ -194,7 +198,12 @@ def dispatchSPARQLQuery(raw_sparql_query, loader, requestArgs, acceptHeader, con glogger.debug('Sending HTTP request to SPARQL endpoint with params: {}'.format(data)) glogger.debug('Sending HTTP request to SPARQL endpoint with headers: {}'.format(reqHeaders)) glogger.debug('Sending HTTP request to SPARQL endpoint with auth: {}'.format(auth)) - response = requests.get(endpoint, params=data, headers=reqHeaders, auth=auth) + try: + response = requests.get(endpoint, params=data, headers=reqHeaders, auth=auth) + except Exception as e: + # Error contacting SPARQL endpoint + glogger.debug('Exception encountered while connecting to SPARQL endpoint') + return { 'error': str(e) }, 400, headers glogger.debug('Response header from endpoint: ' + response.headers['Content-Type']) # Response headers @@ -212,7 +221,7 @@ def dispatchSPARQLQuery(raw_sparql_query, loader, requestArgs, acceptHeader, con if 'proto' in query_metadata: # sparql transformer resp = SPARQLTransformer.post_process(json.loads(resp), query_metadata['proto'], query_metadata['opt']) - if 'transform' in query_metadata: # sparql transformer + if 'transform' in query_metadata and acceptHeader == 'application/json': # sparql transformer rq = { 'proto': query_metadata['transform'] } _, _, opt = SPARQLTransformer.pre_process(rq) resp = SPARQLTransformer.post_process(json.loads(resp), query_metadata['transform'], opt)