How to use resource_id method in wpt

Best JavaScript code snippet using wpt

test_search.py

Source:test_search.py

1# encoding: utf-82import json3import nose4import urllib5import pprint6import sqlalchemy.orm as orm7import ckan.plugins as p8import ckan.lib.create_test_data as ctd9import ckan.model as model10import ckan.tests.legacy as tests11from ckan.common import config12import ckanext.datastore.backend.postgres as db13from ckanext.datastore.tests.helpers import (14 extract, rebuild_all_dbs,15 DatastoreFunctionalTestBase, DatastoreLegacyTestBase)16import ckan.tests.helpers as helpers17import ckan.tests.factories as factories18assert_equals = nose.tools.assert_equals19assert_raises = nose.tools.assert_raises20assert_in = nose.tools.assert_in21class TestDatastoreSearchNewTest(DatastoreFunctionalTestBase):22 def test_fts_on_field_calculates_ranks_only_on_that_specific_field(self):23 resource = factories.Resource()24 data = {25 'resource_id': resource['id'],26 'force': True,27 'records': [28 {'from': 'Brazil', 'to': 'Brazil'},29 {'from': 'Brazil', 'to': 'Italy'}30 ],31 }32 result = helpers.call_action('datastore_create', **data)33 search_data = {34 'resource_id': resource['id'],35 'fields': 'from',36 'q': {37 'from': 'Brazil'38 },39 }40 result = helpers.call_action('datastore_search', **search_data)41 ranks = [r['rank from'] for r in result['records']]42 assert_equals(len(result['records']), 2)43 assert_equals(len(set(ranks)), 1)44 def test_fts_works_on_non_textual_fields(self):45 resource = factories.Resource()46 data = {47 'resource_id': resource['id'],48 'force': True,49 'records': [50 {'from': 'Brazil', 'year': {'foo': 2014}},51 {'from': 'Brazil', 'year': {'foo': 1986}}52 ],53 }54 result = helpers.call_action('datastore_create', **data)55 search_data = {56 'resource_id': resource['id'],57 'fields': 'year',58 'plain': False,59 'q': {60 'year': '20:*'61 },62 }63 result = helpers.call_action('datastore_search', **search_data)64 assert_equals(len(result['records']), 1)65 assert_equals(result['records'][0]['year'], {'foo': 2014})66 def test_all_params_work_with_fields_with_whitespaces(self):67 resource = factories.Resource()68 data = {69 'resource_id': resource['id'],70 'force': True,71 'records': [72 {'the year': 2014},73 {'the year': 2013},74 ],75 }76 result = helpers.call_action('datastore_create', **data)77 search_data = {78 'resource_id': resource['id'],79 'fields': 'the year',80 'sort': 'the year',81 'filters': {82 'the year': 201383 },84 'q': {85 'the year': '2013'86 },87 }88 result = helpers.call_action('datastore_search', **search_data)89 result_years = [r['the year'] for r in result['records']]90 assert_equals(result_years, [2013])91 def test_search_without_total(self):92 resource = factories.Resource()93 data = {94 'resource_id': resource['id'],95 'force': True,96 'records': [97 {'the year': 2014},98 {'the year': 2013},99 ],100 }101 result = helpers.call_action('datastore_create', **data)102 search_data = {103 'resource_id': resource['id'],104 'include_total': False105 }106 result = helpers.call_action('datastore_search', **search_data)107 assert 'total' not in result108class TestDatastoreSearch(DatastoreLegacyTestBase):109 sysadmin_user = None110 normal_user = None111 @classmethod112 def setup_class(cls):113 cls.app = helpers._get_test_app()114 super(TestDatastoreSearch, cls).setup_class()115 ctd.CreateTestData.create()116 cls.sysadmin_user = model.User.get('testsysadmin')117 cls.normal_user = model.User.get('annafan')118 cls.dataset = model.Package.get('annakarenina')119 cls.resource = cls.dataset.resources[0]120 cls.data = {121 'resource_id': cls.resource.id,122 'force': True,123 'aliases': 'books3',124 'fields': [{'id': u'b\xfck', 'type': 'text'},125 {'id': 'author', 'type': 'text'},126 {'id': 'published'},127 {'id': u'characters', u'type': u'_text'},128 {'id': 'rating with %'}],129 'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',130 'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}],131 u'characters': [u'Princess Anna', u'Sergius'],132 'rating with %': '60%'},133 {u'b\xfck': 'warandpeace', 'author': 'tolstoy',134 'nested': {'a': 'b'}, 'rating with %': '99%'}135 ]136 }137 postparams = '%s=1' % json.dumps(cls.data)138 auth = {'Authorization': str(cls.sysadmin_user.apikey)}139 res = cls.app.post('/api/action/datastore_create', params=postparams,140 extra_environ=auth)141 res_dict = json.loads(res.body)142 assert res_dict['success'] is True143 # Make an organization, because private datasets must belong to one.144 cls.organization = tests.call_action_api(145 cls.app, 'organization_create',146 name='test_org',147 apikey=cls.sysadmin_user.apikey)148 cls.expected_records = [{u'published': u'2005-03-01T00:00:00',149 u'_id': 1,150 u'nested': [u'b', {u'moo': u'moo'}],151 u'b\xfck': u'annakarenina',152 u'author': u'tolstoy',153 u'characters': [u'Princess Anna', u'Sergius'],154 u'rating with %': u'60%'},155 {u'published': None,156 u'_id': 2,157 u'nested': {u'a': u'b'},158 u'b\xfck': u'warandpeace',159 u'author': u'tolstoy',160 u'characters': None,161 u'rating with %': u'99%'}]162 engine = db.get_write_engine()163 cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))164 def test_search_basic(self):165 data = {'resource_id': self.data['resource_id']}166 postparams = '%s=1' % json.dumps(data)167 auth = {'Authorization': str(self.normal_user.apikey)}168 res = self.app.post('/api/action/datastore_search', params=postparams,169 extra_environ=auth)170 res_dict = json.loads(res.body)171 assert res_dict['success'] is True172 result = res_dict['result']173 assert result['total'] == len(self.data['records'])174 assert result['records'] == self.expected_records, result['records']175 # search with parameter id should yield the same results176 data = {'id': self.data['resource_id']}177 postparams = '%s=1' % json.dumps(data)178 auth = {'Authorization': str(self.normal_user.apikey)}179 res = self.app.post('/api/action/datastore_search', params=postparams,180 extra_environ=auth)181 res_dict = json.loads(res.body)182 assert res_dict['success'] is True183 result = res_dict['result']184 assert result['total'] == len(self.data['records'])185 assert result['records'] == self.expected_records, result['records']186 def test_search_private_dataset(self):187 group = self.dataset.get_groups()[0]188 context = {189 'user': self.sysadmin_user.name,190 'ignore_auth': True,191 'model': model}192 package = p.toolkit.get_action('package_create')(193 context,194 {'name': 'privatedataset',195 'private': True,196 'owner_org': self.organization['id'],197 'groups': [{198 'id': group.id199 }]})200 resource = p.toolkit.get_action('resource_create')(201 context,202 {'name': 'privateresource',203 'url': 'https://www.example.com/',204 'package_id': package['id']})205 postparams = '%s=1' % json.dumps({206 'resource_id': resource['id'],207 'force': True208 })209 auth = {'Authorization': str(self.sysadmin_user.apikey)}210 res = self.app.post('/api/action/datastore_create', params=postparams,211 extra_environ=auth)212 res_dict = json.loads(res.body)213 assert res_dict['success'] is True214 data = {'resource_id': resource['id']}215 postparams = '%s=1' % json.dumps(data)216 auth = {'Authorization': str(self.normal_user.apikey)}217 res = self.app.post('/api/action/datastore_search', params=postparams,218 extra_environ=auth, status=403)219 res_dict = json.loads(res.body)220 assert res_dict['success'] is False221 def test_search_alias(self):222 data = {'resource_id': self.data['aliases']}223 postparams = '%s=1' % json.dumps(data)224 auth = {'Authorization': str(self.normal_user.apikey)}225 res = self.app.post('/api/action/datastore_search', params=postparams,226 extra_environ=auth)227 res_dict_alias = json.loads(res.body)228 result = res_dict_alias['result']229 assert result['total'] == len(self.data['records'])230 assert result['records'] == self.expected_records, result['records']231 def test_search_invalid_field(self):232 data = {'resource_id': self.data['resource_id'],233 'fields': [{'id': 'bad'}]}234 postparams = '%s=1' % json.dumps(data)235 auth = {'Authorization': str(self.normal_user.apikey)}236 res = self.app.post('/api/action/datastore_search', params=postparams,237 extra_environ=auth, status=409)238 res_dict = json.loads(res.body)239 assert res_dict['success'] is False240 def test_search_fields(self):241 data = {'resource_id': self.data['resource_id'],242 'fields': [u'b\xfck']}243 postparams = '%s=1' % json.dumps(data)244 auth = {'Authorization': str(self.normal_user.apikey)}245 res = self.app.post('/api/action/datastore_search', params=postparams,246 extra_environ=auth)247 res_dict = json.loads(res.body)248 assert res_dict['success'] is True249 result = res_dict['result']250 assert result['total'] == len(self.data['records'])251 assert result['records'] == [{u'b\xfck': 'annakarenina'},252 {u'b\xfck': 'warandpeace'}], result['records']253 data = {'resource_id': self.data['resource_id'],254 'fields': u'b\xfck, author'}255 postparams = '%s=1' % json.dumps(data)256 auth = {'Authorization': str(self.normal_user.apikey)}257 res = self.app.post('/api/action/datastore_search', params=postparams,258 extra_environ=auth)259 res_dict = json.loads(res.body)260 assert res_dict['success'] is True261 result = res_dict['result']262 assert result['total'] == len(self.data['records'])263 assert result['records'] == [{u'b\xfck': 'annakarenina', 'author': 'tolstoy'},264 {u'b\xfck': 'warandpeace', 'author': 'tolstoy'}], result['records']265 def test_search_distinct(self):266 data = {'resource_id': self.data['resource_id'],267 'fields': [u'author'],268 'distinct': True}269 postparams = '%s=1' % json.dumps(data)270 auth = {'Authorization': str(self.normal_user.apikey)}271 res = self.app.post('/api/action/datastore_search', params=postparams,272 extra_environ=auth)273 res_dict = json.loads(res.body)274 assert res_dict['success'] is True275 result = res_dict['result']276 assert result['total'] == 1277 assert result['records'] == [{u'author': 'tolstoy'}], result['records']278 def test_search_filters(self):279 data = {'resource_id': self.data['resource_id'],280 'filters': {u'b\xfck': 'annakarenina'}}281 postparams = '%s=1' % json.dumps(data)282 auth = {'Authorization': str(self.normal_user.apikey)}283 res = self.app.post('/api/action/datastore_search', params=postparams,284 extra_environ=auth)285 res_dict = json.loads(res.body)286 assert res_dict['success'] is True287 result = res_dict['result']288 assert result['total'] == 1289 assert result['records'] == [self.expected_records[0]]290 def test_search_filter_array_field(self):291 data = {'resource_id': self.data['resource_id'],292 'filters': {u'characters': [u'Princess Anna', u'Sergius']}}293 postparams = '%s=1' % json.dumps(data)294 auth = {'Authorization': str(self.normal_user.apikey)}295 res = self.app.post('/api/action/datastore_search', params=postparams,296 extra_environ=auth)297 res_dict = json.loads(res.body)298 assert res_dict['success'] is True299 result = res_dict['result']300 assert result['total'] == 1301 assert_equals(result['records'], [self.expected_records[0]])302 def test_search_multiple_filters_on_same_field(self):303 data = {'resource_id': self.data['resource_id'],304 'filters': {305 u'b\xfck': [u'annakarenina', u'warandpeace']306 }}307 postparams = '%s=1' % json.dumps(data)308 auth = {'Authorization': str(self.normal_user.apikey)}309 res = self.app.post('/api/action/datastore_search', params=postparams,310 extra_environ=auth)311 res_dict = json.loads(res.body)312 assert res_dict['success'] is True313 result = res_dict['result']314 assert result['total'] == 2315 assert_equals(result['records'], self.expected_records)316 def test_search_filter_normal_field_passing_multiple_values_in_array(self):317 data = {'resource_id': self.data['resource_id'],318 'filters': {u'b\xfck': [u'annakarenina', u'warandpeace']}}319 postparams = '%s=1' % json.dumps(data)320 auth = {'Authorization': str(self.normal_user.apikey)}321 res = self.app.post('/api/action/datastore_search', params=postparams,322 extra_environ=auth)323 res_dict = json.loads(res.body)324 assert res_dict['success'] is True325 result = res_dict['result']326 assert result['total'] == 2327 assert result['records'] == self.expected_records, result['records']328 def test_search_filters_get(self):329 filters = {u'b\xfck': 'annakarenina'}330 res = self.app.get('/api/action/datastore_search?resource_id={0}&filters={1}'.format(331 self.data['resource_id'], json.dumps(filters)))332 res_dict = json.loads(res.body)333 assert res_dict['success'] is True334 result = res_dict['result']335 assert result['total'] == 1336 assert result['records'] == [self.expected_records[0]]337 def test_search_invalid_filter(self):338 data = {'resource_id': self.data['resource_id'],339 # invalid because author is not a numeric field340 'filters': {u'author': 42}}341 postparams = '%s=1' % json.dumps(data)342 auth = {'Authorization': str(self.sysadmin_user.apikey)}343 res = self.app.post('/api/action/datastore_search', params=postparams,344 extra_environ=auth, status=409)345 res_dict = json.loads(res.body)346 assert res_dict['success'] is False347 def test_search_sort(self):348 data = {'resource_id': self.data['resource_id'],349 'sort': u'b\xfck asc, author desc'}350 postparams = '%s=1' % json.dumps(data)351 auth = {'Authorization': str(self.normal_user.apikey)}352 res = self.app.post('/api/action/datastore_search', params=postparams,353 extra_environ=auth)354 res_dict = json.loads(res.body)355 assert res_dict['success'] is True356 result = res_dict['result']357 assert result['total'] == 2358 assert result['records'] == self.expected_records, result['records']359 data = {'resource_id': self.data['resource_id'],360 'sort': [u'b\xfck desc', '"author" asc']}361 postparams = '%s=1' % json.dumps(data)362 res = self.app.post('/api/action/datastore_search', params=postparams,363 extra_environ=auth)364 res_dict = json.loads(res.body)365 assert res_dict['success'] is True366 result = res_dict['result']367 assert result['total'] == 2368 assert result['records'] == self.expected_records[::-1]369 def test_search_invalid(self):370 data = {'resource_id': self.data['resource_id'],371 'sort': u'f\xfc\xfc asc'}372 postparams = '%s=1' % json.dumps(data)373 auth = {'Authorization': str(self.sysadmin_user.apikey)}374 res = self.app.post('/api/action/datastore_search', params=postparams,375 extra_environ=auth, status=409)376 res_dict = json.loads(res.body)377 assert res_dict['success'] is False378 error_msg = res_dict['error']['sort'][0]379 assert u'f\xfc\xfc' in error_msg, \380 'Expected "{0}" to contain "{1}"'.format(error_msg, u'f\xfc\xfc')381 def test_search_limit(self):382 data = {'resource_id': self.data['resource_id'],383 'limit': 1}384 postparams = '%s=1' % json.dumps(data)385 auth = {'Authorization': str(self.normal_user.apikey)}386 res = self.app.post('/api/action/datastore_search', params=postparams,387 extra_environ=auth)388 res_dict = json.loads(res.body)389 assert res_dict['success'] is True390 result = res_dict['result']391 assert result['total'] == 2392 assert result['records'] == [self.expected_records[0]]393 def test_search_invalid_limit(self):394 data = {'resource_id': self.data['resource_id'],395 'limit': 'bad'}396 postparams = '%s=1' % json.dumps(data)397 auth = {'Authorization': str(self.normal_user.apikey)}398 res = self.app.post('/api/action/datastore_search', params=postparams,399 extra_environ=auth, status=409)400 res_dict = json.loads(res.body)401 assert res_dict['success'] is False402 data = {'resource_id': self.data['resource_id'],403 'limit': -1}404 postparams = '%s=1' % json.dumps(data)405 auth = {'Authorization': str(self.normal_user.apikey)}406 res = self.app.post('/api/action/datastore_search', params=postparams,407 extra_environ=auth, status=409)408 res_dict = json.loads(res.body)409 assert res_dict['success'] is False410 def test_search_offset(self):411 data = {'resource_id': self.data['resource_id'],412 'limit': 1,413 'offset': 1}414 postparams = '%s=1' % json.dumps(data)415 auth = {'Authorization': str(self.normal_user.apikey)}416 res = self.app.post('/api/action/datastore_search', params=postparams,417 extra_environ=auth)418 res_dict = json.loads(res.body)419 assert res_dict['success'] is True420 result = res_dict['result']421 assert result['total'] == 2422 assert result['records'] == [self.expected_records[1]]423 def test_search_invalid_offset(self):424 data = {'resource_id': self.data['resource_id'],425 'offset': 'bad'}426 postparams = '%s=1' % json.dumps(data)427 auth = {'Authorization': str(self.normal_user.apikey)}428 res = self.app.post('/api/action/datastore_search', params=postparams,429 extra_environ=auth, status=409)430 res_dict = json.loads(res.body)431 assert res_dict['success'] is False432 data = {'resource_id': self.data['resource_id'],433 'offset': -1}434 postparams = '%s=1' % json.dumps(data)435 auth = {'Authorization': str(self.normal_user.apikey)}436 res = self.app.post('/api/action/datastore_search', params=postparams,437 extra_environ=auth, status=409)438 res_dict = json.loads(res.body)439 assert res_dict['success'] is False440 def test_search_full_text(self):441 data = {'resource_id': self.data['resource_id'],442 'q': 'annakarenina'}443 postparams = '%s=1' % json.dumps(data)444 auth = {'Authorization': str(self.normal_user.apikey)}445 res = self.app.post('/api/action/datastore_search', params=postparams,446 extra_environ=auth)447 res_dict = json.loads(res.body)448 assert res_dict['success'] is True449 result = res_dict['result']450 assert result['total'] == 1451 results = [extract(result['records'][0], [452 u'_id', u'author', u'b\xfck', u'nested',453 u'published', u'characters', u'rating with %'])]454 assert results == [self.expected_records[0]], results['records']455 data = {'resource_id': self.data['resource_id'],456 'q': 'tolstoy'}457 postparams = '%s=1' % json.dumps(data)458 res = self.app.post('/api/action/datastore_search', params=postparams,459 extra_environ=auth)460 res_dict = json.loads(res.body)461 assert res_dict['success'] is True462 result = res_dict['result']463 assert result['total'] == 2464 results = [extract(465 record,466 [u'_id', u'author', u'b\xfck', u'nested',467 u'published', u'characters', u'rating with %']468 ) for record in result['records']]469 assert results == self.expected_records, result['records']470 expected_fields = [{u'type': u'int', u'id': u'_id'},471 {u'type': u'text', u'id': u'b\xfck'},472 {u'type': u'text', u'id': u'author'},473 {u'type': u'timestamp', u'id': u'published'},474 {u'type': u'json', u'id': u'nested'}]475 for field in expected_fields:476 assert_in(field, result['fields'])477 # test multiple word queries (connected with and)478 data = {'resource_id': self.data['resource_id'],479 'plain': True,480 'q': 'tolstoy annakarenina'}481 postparams = '%s=1' % json.dumps(data)482 res = self.app.post('/api/action/datastore_search', params=postparams,483 extra_environ=auth)484 res_dict = json.loads(res.body)485 assert res_dict['success'] is True486 result = res_dict['result']487 assert result['total'] == 1488 results = [extract(489 result['records'][0],490 [u'_id', u'author', u'b\xfck', u'nested', u'published',491 u'characters', u'rating with %'])]492 assert results == [self.expected_records[0]], results['records']493 for field in expected_fields:494 assert field in result['fields'], field495 def test_search_full_text_on_specific_column(self):496 data = {'resource_id': self.data['resource_id'],497 'q': {u"b\xfck": "annakarenina"}498 }499 postparams = '%s=1' % json.dumps(data)500 auth = {'Authorization': str(self.normal_user.apikey)}501 res = self.app.post('/api/action/datastore_search', params=postparams,502 extra_environ=auth)503 res_dict = json.loads(res.body)504 assert res_dict['success'] is True505 assert_equals(len(res_dict['result']['records']), 1)506 assert_equals(res_dict['result']['records'][0]['_id'],507 self.expected_records[0]['_id'])508 def test_search_full_text_on_specific_column_even_if_q_is_a_json_string(self):509 data = {'resource_id': self.data['resource_id'],510 'q': u'{"b\xfck": "annakarenina"}'511 }512 postparams = '%s=1' % json.dumps(data)513 auth = {'Authorization': str(self.normal_user.apikey)}514 res = self.app.post('/api/action/datastore_search', params=postparams,515 extra_environ=auth)516 res_dict = json.loads(res.body)517 assert res_dict['success'] is True518 assert_equals(len(res_dict['result']['records']), 1)519 assert_equals(res_dict['result']['records'][0]['_id'],520 self.expected_records[0]['_id'])521 def test_search_full_text_invalid_field_name(self):522 data = {'resource_id': self.data['resource_id'],523 'q': {'invalid_field_name': 'value'}524 }525 postparams = '%s=1' % json.dumps(data)526 auth = {'Authorization': str(self.normal_user.apikey)}527 res = self.app.post('/api/action/datastore_search', params=postparams,528 extra_environ=auth, status=409)529 res_dict = json.loads(res.body)530 assert res_dict['success'] is False531 def test_search_full_text_invalid_field_value(self):532 data = {'resource_id': self.data['resource_id'],533 'q': {'author': ['invalid', 'value']}534 }535 postparams = '%s=1' % json.dumps(data)536 auth = {'Authorization': str(self.normal_user.apikey)}537 res = self.app.post('/api/action/datastore_search', params=postparams,538 extra_environ=auth, status=409)539 res_dict = json.loads(res.body)540 assert res_dict['success'] is False541 def test_search_table_metadata(self):542 data = {'resource_id': "_table_metadata"}543 postparams = '%s=1' % json.dumps(data)544 auth = {'Authorization': str(self.normal_user.apikey)}545 res = self.app.post('/api/action/datastore_search', params=postparams,546 extra_environ=auth)547 res_dict = json.loads(res.body)548 assert res_dict['success'] is True549 def test_search_is_unsuccessful_when_called_with_filters_not_as_dict(self):550 data = {551 'resource_id': self.data['resource_id'],552 'filters': 'the-filter'553 }554 postparams = '%s=1' % json.dumps(data)555 auth = {'Authorization': str(self.normal_user.apikey)}556 res = self.app.post('/api/action/datastore_search', params=postparams,557 extra_environ=auth, status=409)558 res_dict = json.loads(res.body)559 assert res_dict['success'] is False560 assert res_dict['error'].get('filters') is not None, res_dict['error']561 def test_search_is_unsuccessful_when_called_with_invalid_filters(self):562 data = {563 'resource_id': self.data['resource_id'],564 'filters': {565 'invalid-column-name': 'value'566 }567 }568 postparams = '%s=1' % json.dumps(data)569 auth = {'Authorization': str(self.normal_user.apikey)}570 res = self.app.post('/api/action/datastore_search', params=postparams,571 extra_environ=auth, status=409)572 res_dict = json.loads(res.body)573 assert res_dict['success'] is False574 assert res_dict['error'].get('filters') is not None, res_dict['error']575 def test_search_is_unsuccessful_when_called_with_invalid_fields(self):576 data = {577 'resource_id': self.data['resource_id'],578 'fields': [579 'invalid-column-name'580 ]581 }582 postparams = '%s=1' % json.dumps(data)583 auth = {'Authorization': str(self.normal_user.apikey)}584 res = self.app.post('/api/action/datastore_search', params=postparams,585 extra_environ=auth, status=409)586 res_dict = json.loads(res.body)587 assert res_dict['success'] is False588 assert res_dict['error'].get('fields') is not None, res_dict['error']589class TestDatastoreFullTextSearch(DatastoreLegacyTestBase):590 @classmethod591 def setup_class(cls):592 cls.app = helpers._get_test_app()593 super(TestDatastoreFullTextSearch, cls).setup_class()594 ctd.CreateTestData.create()595 cls.sysadmin_user = model.User.get('testsysadmin')596 cls.normal_user = model.User.get('annafan')597 resource = model.Package.get('annakarenina').resources[0]598 cls.data = dict(599 resource_id=resource.id,600 force=True,601 fields=[602 {'id': 'id'},603 {'id': 'date', 'type':'date'},604 {'id': 'x'},605 {'id': 'y'},606 {'id': 'z'},607 {'id': 'country'},608 {'id': 'title'},609 {'id': 'lat'},610 {'id': 'lon'}611 ],612 records=[613 {'id': 0, 'date': '2011-01-01', 'x': 1, 'y': 2, 'z': 3, 'country': 'DE', 'title': 'first 99', 'lat':52.56, 'lon':13.40},614 {'id': 1, 'date': '2011-02-02', 'x': 2, 'y': 4, 'z': 24, 'country': 'UK', 'title': 'second', 'lat':54.97, 'lon':-1.60},615 {'id': 2, 'date': '2011-03-03', 'x': 3, 'y': 6, 'z': 9, 'country': 'US', 'title': 'third', 'lat':40.00, 'lon':-75.5},616 {'id': 3, 'date': '2011-04-04', 'x': 4, 'y': 8, 'z': 6, 'country': 'UK', 'title': 'fourth', 'lat':57.27, 'lon':-6.20},617 {'id': 4, 'date': '2011-05-04', 'x': 5, 'y': 10, 'z': 15, 'country': 'UK', 'title': 'fifth', 'lat':51.58, 'lon':0},618 {'id': 5, 'date': '2011-06-02', 'x': 6, 'y': 12, 'z': 18, 'country': 'DE', 'title': 'sixth 53.56', 'lat':51.04, 'lon':7.9}619 ]620 )621 postparams = '%s=1' % json.dumps(cls.data)622 auth = {'Authorization': str(cls.normal_user.apikey)}623 res = cls.app.post('/api/action/datastore_create', params=postparams,624 extra_environ=auth)625 res_dict = json.loads(res.body)626 assert res_dict['success'] is True627 def test_search_full_text(self):628 data = {'resource_id': self.data['resource_id'],629 'q': 'DE'}630 postparams = '%s=1' % json.dumps(data)631 auth = {'Authorization': str(self.normal_user.apikey)}632 res = self.app.post('/api/action/datastore_search', params=postparams,633 extra_environ=auth)634 res_dict = json.loads(res.body)635 assert res_dict['result']['total'] == 2, pprint.pformat(res_dict)636 def test_advanced_search_full_text(self):637 data = {'resource_id': self.data['resource_id'],638 'plain': 'False',639 'q': 'DE | UK'}640 postparams = '%s=1' % json.dumps(data)641 auth = {'Authorization': str(self.normal_user.apikey)}642 res = self.app.post('/api/action/datastore_search', params=postparams,643 extra_environ=auth)644 res_dict = json.loads(res.body)645 assert res_dict['result']['total'] == 5, pprint.pformat(res_dict)646 def test_full_text_search_on_integers_within_text_strings(self):647 data = {'resource_id': self.data['resource_id'],648 'q': '99'}649 postparams = '%s=1' % json.dumps(data)650 auth = {'Authorization': str(self.normal_user.apikey)}651 res = self.app.post('/api/action/datastore_search', params=postparams,652 extra_environ=auth)653 res_dict = json.loads(res.body)654 assert res_dict['result']['total'] == 1, pprint.pformat(res_dict)655 def test_full_text_search_on_integers(self):656 data = {'resource_id': self.data['resource_id'],657 'q': '4'}658 postparams = '%s=1' % json.dumps(data)659 auth = {'Authorization': str(self.normal_user.apikey)}660 res = self.app.post('/api/action/datastore_search', params=postparams,661 extra_environ=auth)662 res_dict = json.loads(res.body)663 assert res_dict['result']['total'] == 3, pprint.pformat(res_dict)664 def test_full_text_search_on_decimal_within_text_strings(self):665 data = {'resource_id': self.data['resource_id'],666 'q': '53.56'}667 postparams = '%s=1' % json.dumps(data)668 auth = {'Authorization': str(self.normal_user.apikey)}669 res = self.app.post('/api/action/datastore_search', params=postparams,670 extra_environ=auth)671 res_dict = json.loads(res.body)672 assert res_dict['result']['total'] == 1, pprint.pformat(res_dict)673 def test_full_text_search_on_decimal(self):674 data = {'resource_id': self.data['resource_id'],675 'q': '52.56'}676 postparams = '%s=1' % json.dumps(data)677 auth = {'Authorization': str(self.normal_user.apikey)}678 res = self.app.post('/api/action/datastore_search', params=postparams,679 extra_environ=auth)680 res_dict = json.loads(res.body)681 assert res_dict['result']['total'] == 1, pprint.pformat(res_dict)682 def test_full_text_search_on_date(self):683 data = {'resource_id': self.data['resource_id'],684 'q': '2011-01-01'}685 postparams = '%s=1' % json.dumps(data)686 auth = {'Authorization': str(self.normal_user.apikey)}687 res = self.app.post('/api/action/datastore_search', params=postparams,688 extra_environ=auth)689 res_dict = json.loads(res.body)690 assert res_dict['result']['total'] == 1, pprint.pformat(res_dict)691 def test_full_text_search_on_json_like_string_succeeds(self):692 data = {'resource_id': self.data['resource_id'],693 'q': '"{}"'}694 postparams = '%s=1' % json.dumps(data)695 auth = {'Authorization': str(self.normal_user.apikey)}696 res = self.app.post('/api/action/datastore_search', params=postparams,697 extra_environ=auth)698 res_dict = json.loads(res.body)699 assert res_dict['success'], pprint.pformat(res_dict)700class TestDatastoreSQL(DatastoreLegacyTestBase):701 sysadmin_user = None702 normal_user = None703 @classmethod704 def setup_class(cls):705 cls.app = helpers._get_test_app()706 super(TestDatastoreSQL, cls).setup_class()707 ctd.CreateTestData.create()708 cls.sysadmin_user = model.User.get('testsysadmin')709 cls.normal_user = model.User.get('annafan')710 cls.dataset = model.Package.get('annakarenina')711 resource = cls.dataset.resources[0]712 cls.data = {713 'resource_id': resource.id,714 'force': True,715 'aliases': 'books4',716 'fields': [{'id': u'b\xfck', 'type': 'text'},717 {'id': 'author', 'type': 'text'},718 {'id': 'published'}],719 'records': [{u'b\xfck': 'annakarenina',720 'author': 'tolstoy',721 'published': '2005-03-01',722 'nested': ['b', {'moo': 'moo'}]},723 {u'b\xfck': 'warandpeace',724 'author': 'tolstoy',725 'nested': {'a': 'b'}}]726 }727 postparams = '%s=1' % json.dumps(cls.data)728 auth = {'Authorization': str(cls.sysadmin_user.apikey)}729 res = cls.app.post('/api/action/datastore_create', params=postparams,730 extra_environ=auth)731 res_dict = json.loads(res.body)732 assert res_dict['success'] is True733 # Make an organization, because private datasets must belong to one.734 cls.organization = tests.call_action_api(735 cls.app, 'organization_create',736 name='test_org',737 apikey=cls.sysadmin_user.apikey)738 cls.expected_records = [{u'_full_text': [u"'annakarenina'", u"'b'",739 u"'moo'", u"'tolstoy'",740 u"'2005'"],741 u'_id': 1,742 u'author': u'tolstoy',743 u'b\xfck': u'annakarenina',744 u'nested': [u'b', {u'moo': u'moo'}],745 u'published': u'2005-03-01T00:00:00'},746 {u'_full_text': [u"'tolstoy'", u"'warandpeac'",747 u"'b'"],748 u'_id': 2,749 u'author': u'tolstoy',750 u'b\xfck': u'warandpeace',751 u'nested': {u'a': u'b'},752 u'published': None}]753 cls.expected_join_results = [{u'first': 1, u'second': 1}, {u'first': 1, u'second': 2}]754 engine = db.get_write_engine()755 cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))756 def test_validates_sql_has_a_single_statement(self):757 sql = 'SELECT * FROM public."{0}"; SELECT * FROM public."{0}";'.format(self.data['resource_id'])758 assert_raises(p.toolkit.ValidationError,759 helpers.call_action, 'datastore_search_sql', sql=sql)760 def test_works_with_semicolons_inside_strings(self):761 sql = 'SELECT * FROM public."{0}" WHERE "author" = \'foo; bar\''.format(self.data['resource_id'])762 helpers.call_action('datastore_search_sql', sql=sql)763 def test_invalid_statement(self):764 query = 'SELECT ** FROM foobar'765 data = {'sql': query}766 postparams = json.dumps(data)767 auth = {'Authorization': str(self.normal_user.apikey)}768 res = self.app.post('/api/action/datastore_search_sql', params=postparams,769 extra_environ=auth, status=409)770 res_dict = json.loads(res.body)771 assert res_dict['success'] is False772 def test_select_basic(self):773 query = 'SELECT * FROM "{0}"'.format(self.data['resource_id'])774 data = {'sql': query}775 postparams = json.dumps(data)776 auth = {'Authorization': str(self.normal_user.apikey)}777 res = self.app.post('/api/action/datastore_search_sql', params=postparams,778 extra_environ=auth)779 res_dict = json.loads(res.body)780 assert res_dict['success'] is True781 result = res_dict['result']782 assert len(result['records']) == len(self.expected_records)783 for (row_index, row) in enumerate(result['records']):784 expected_row = self.expected_records[row_index]785 assert set(row.keys()) == set(expected_row.keys())786 for field in row:787 if field == '_full_text':788 for ft_value in expected_row['_full_text']:789 assert ft_value in row['_full_text']790 else:791 assert row[field] == expected_row[field]792 # test alias search793 query = 'SELECT * FROM "{0}"'.format(self.data['aliases'])794 data = {'sql': query}795 postparams = json.dumps(data)796 res = self.app.post('/api/action/datastore_search_sql', params=postparams,797 extra_environ=auth)798 res_dict_alias = json.loads(res.body)799 assert result['records'] == res_dict_alias['result']['records']800 def test_select_where_like_with_percent(self):801 query = 'SELECT * FROM public."{0}" WHERE "author" LIKE \'tol%\''.format(self.data['resource_id'])802 data = {'sql': query}803 postparams = json.dumps(data)804 auth = {'Authorization': str(self.sysadmin_user.apikey)}805 res = self.app.post('/api/action/datastore_search_sql', params=postparams,806 extra_environ=auth)807 res_dict = json.loads(res.body)808 assert res_dict['success'] is True809 result = res_dict['result']810 assert len(result['records']) == len(self.expected_records)811 for (row_index, row) in enumerate(result['records']):812 expected_row = self.expected_records[row_index]813 assert set(row.keys()) == set(expected_row.keys())814 for field in row:815 if field == '_full_text':816 for ft_value in expected_row['_full_text']:817 assert ft_value in row['_full_text']818 else:819 assert row[field] == expected_row[field]820 def test_self_join(self):821 query = '''822 select a._id as first, b._id as second823 from "{0}" AS a,824 "{0}" AS b825 where a.author = b.author826 limit 2827 '''.format(self.data['resource_id'])828 data = urllib.urlencode({'sql': query})829 auth = {'Authorization': str(self.normal_user.apikey)}830 res = self.app.post('/api/action/datastore_search_sql', params=data,831 extra_environ=auth)832 res_dict = json.loads(res.body)833 assert res_dict['success'] is True834 result = res_dict['result']835 assert result['records'] == self.expected_join_results836 def test_new_datastore_table_from_private_resource(self):837 # make a private CKAN resource838 group = self.dataset.get_groups()[0]839 context = {840 'user': self.sysadmin_user.name,841 'ignore_auth': True,842 'model': model}843 package = p.toolkit.get_action('package_create')(844 context,845 {'name': 'privatedataset',846 'private': True,847 'owner_org': self.organization['id'],848 'groups': [{849 'id': group.id850 }]})851 resource = p.toolkit.get_action('resource_create')(852 context,853 {'name': 'privateresource',854 'url': 'https://www.example.com/',855 'package_id': package['id']})856 postparams = '%s=1' % json.dumps({857 'resource_id': resource['id'],858 'force': True859 })860 auth = {'Authorization': str(self.sysadmin_user.apikey)}861 res = self.app.post('/api/action/datastore_create', params=postparams,862 extra_environ=auth)863 res_dict = json.loads(res.body)864 assert res_dict['success'] is True865 # new resource should be private866 query = 'SELECT * FROM "{0}"'.format(resource['id'])867 data = {'sql': query}868 postparams = json.dumps(data)869 auth = {'Authorization': str(self.normal_user.apikey)}870 res = self.app.post('/api/action/datastore_search_sql', params=postparams,871 extra_environ=auth, status=403)872 res_dict = json.loads(res.body)873 assert res_dict['success'] is False874 assert res_dict['error']['__type'] == 'Authorization Error'875 def test_not_authorized_to_access_system_tables(self):876 test_cases = [877 'SELECT * FROM pg_roles',878 'SELECT * FROM pg_catalog.pg_database',879 'SELECT rolpassword FROM pg_roles',880 '''SELECT p.rolpassword881 FROM pg_roles p882 JOIN "{0}" r883 ON p.rolpassword = r.author'''.format(self.data['resource_id']),884 ]885 for query in test_cases:886 data = {'sql': query.replace('\n', '')}887 postparams = urllib.urlencode(data)888 res = self.app.post('/api/action/datastore_search_sql',889 params=postparams,890 status=403)891 res_dict = json.loads(res.body)892 assert res_dict['success'] is False893 assert res_dict['error']['__type'] == 'Authorization Error'894class TestDatastoreSQLFunctional(DatastoreFunctionalTestBase):895 def test_search_sql_enforces_private(self):896 user1 = factories.User()897 user2 = factories.User()898 user3 = factories.User()899 ctx1 = {u'user': user1['name'], u'ignore_auth': False}900 ctx2 = {u'user': user2['name'], u'ignore_auth': False}901 ctx3 = {u'user': user3['name'], u'ignore_auth': False}902 org1 = factories.Organization(903 user=user1,904 users=[{u'name': user3['name'], u'capacity': u'member'}])905 org2 = factories.Organization(906 user=user2,907 users=[{u'name': user3['name'], u'capacity': u'member'}])908 ds1 = factories.Dataset(owner_org=org1['id'], private=True)909 ds2 = factories.Dataset(owner_org=org2['id'], private=True)910 r1 = helpers.call_action(911 u'datastore_create',912 resource={u'package_id': ds1['id']},913 fields=[{u'id': u'spam', u'type': u'text'}])914 r2 = helpers.call_action(915 u'datastore_create',916 resource={u'package_id': ds2['id']},917 fields=[{u'id': u'ham', u'type': u'text'}])918 sql1 = 'SELECT spam FROM "{0}"'.format(r1['resource_id'])919 sql2 = 'SELECT ham FROM "{0}"'.format(r2['resource_id'])920 sql3 = 'SELECT spam, ham FROM "{0}", "{1}"'.format(921 r1['resource_id'], r2['resource_id'])922 assert_raises(923 p.toolkit.NotAuthorized,924 helpers.call_action,925 'datastore_search_sql',926 context=ctx2,927 sql=sql1)928 assert_raises(929 p.toolkit.NotAuthorized,930 helpers.call_action,931 'datastore_search_sql',932 context=ctx1,933 sql=sql2)934 assert_raises(935 p.toolkit.NotAuthorized,936 helpers.call_action,937 'datastore_search_sql',938 context=ctx1,939 sql=sql3)940 assert_raises(941 p.toolkit.NotAuthorized,942 helpers.call_action,943 'datastore_search_sql',944 context=ctx2,945 sql=sql3)946 helpers.call_action(947 'datastore_search_sql',948 context=ctx1,949 sql=sql1)950 helpers.call_action(951 'datastore_search_sql',952 context=ctx2,953 sql=sql2)954 helpers.call_action(955 'datastore_search_sql',956 context=ctx3,957 sql=sql3)958class TestDatastoreSearchRecordsFormat(DatastoreFunctionalTestBase):959 def test_sort_results_objects(self):960 ds = factories.Dataset()961 r = helpers.call_action(962 u'datastore_create',963 resource={u'package_id': ds['id']},964 fields=[965 {u'id': u'num', u'type': u'numeric'},966 {u'id': u'dt', u'type': u'timestamp'},967 {u'id': u'txt', u'type': u'text'}],968 records=[969 {u'num': 10, u'dt': u'2020-01-01', u'txt': 'aaab'},970 {u'num': 9, u'dt': u'2020-01-02', u'txt': 'aaab'},971 {u'num': 9, u'dt': u'2020-01-01', u'txt': 'aaac'}])972 assert_equals(973 helpers.call_action(974 'datastore_search',975 resource_id=r['resource_id'],976 sort=u'num, dt')['records'],977 [978 {u'_id': 3, u'num': 9, u'dt': u'2020-01-01T00:00:00', u'txt': u'aaac'},979 {u'_id': 2, u'num': 9, u'dt': u'2020-01-02T00:00:00', u'txt': u'aaab'},980 {u'_id': 1, u'num': 10, u'dt': u'2020-01-01T00:00:00', u'txt': u'aaab'},981 ])982 assert_equals(983 helpers.call_action(984 'datastore_search',985 resource_id=r['resource_id'],986 sort=u'dt, txt')['records'],987 [988 {u'_id': 1, u'num': 10, u'dt': u'2020-01-01T00:00:00', u'txt': u'aaab'},989 {u'_id': 3, u'num': 9, u'dt': u'2020-01-01T00:00:00', u'txt': u'aaac'},990 {u'_id': 2, u'num': 9, u'dt': u'2020-01-02T00:00:00', u'txt': u'aaab'},991 ])992 assert_equals(993 helpers.call_action(994 'datastore_search',995 resource_id=r['resource_id'],996 sort=u'txt, num')['records'],997 [998 {u'_id': 2, u'num': 9, u'dt': u'2020-01-02T00:00:00', u'txt': u'aaab'},999 {u'_id': 1, u'num': 10, u'dt': u'2020-01-01T00:00:00', u'txt': u'aaab'},1000 {u'_id': 3, u'num': 9, u'dt': u'2020-01-01T00:00:00', u'txt': u'aaac'},1001 ])1002 def test_sort_results_lists(self):1003 ds = factories.Dataset()1004 r = helpers.call_action(1005 u'datastore_create',1006 resource={u'package_id': ds['id']},1007 fields=[1008 {u'id': u'num', u'type': u'numeric'},1009 {u'id': u'dt', u'type': u'timestamp'},1010 {u'id': u'txt', u'type': u'text'}],1011 records=[1012 {u'num': 10, u'dt': u'2020-01-01', u'txt': u'aaab'},1013 {u'num': 9, u'dt': u'2020-01-02', u'txt': u'aaab'},1014 {u'num': 9, u'dt': u'2020-01-01', u'txt': u'aaac'}])1015 assert_equals(1016 helpers.call_action(1017 'datastore_search',1018 resource_id=r['resource_id'],1019 records_format=u'lists',1020 sort=u'num, dt')['records'],1021 [1022 [3, 9, u'2020-01-01T00:00:00', u'aaac'],1023 [2, 9, u'2020-01-02T00:00:00', u'aaab'],1024 [1, 10, u'2020-01-01T00:00:00', u'aaab'],1025 ])1026 assert_equals(1027 helpers.call_action(1028 'datastore_search',1029 resource_id=r['resource_id'],1030 records_format=u'lists',1031 sort=u'dt, txt')['records'],1032 [1033 [1, 10, u'2020-01-01T00:00:00', u'aaab'],1034 [3, 9, u'2020-01-01T00:00:00', u'aaac'],1035 [2, 9, u'2020-01-02T00:00:00', u'aaab'],1036 ])1037 assert_equals(1038 helpers.call_action(1039 'datastore_search',1040 resource_id=r['resource_id'],1041 records_format=u'lists',1042 sort=u'txt, num')['records'],1043 [1044 [2, 9, u'2020-01-02T00:00:00', u'aaab'],1045 [1, 10, u'2020-01-01T00:00:00', u'aaab'],1046 [3, 9, u'2020-01-01T00:00:00', u'aaac'],1047 ])1048 def test_sort_results_csv(self):1049 ds = factories.Dataset()1050 r = helpers.call_action(1051 u'datastore_create',1052 resource={u'package_id': ds['id']},1053 fields=[1054 {u'id': u'num', u'type': u'numeric'},1055 {u'id': u'dt', u'type': u'timestamp'},1056 {u'id': u'txt', u'type': u'text'}],1057 records=[1058 {u'num': 10, u'dt': u'2020-01-01', u'txt': u'aaab'},1059 {u'num': 9, u'dt': u'2020-01-02', u'txt': u'aaab'},1060 {u'num': 9, u'dt': u'2020-01-01', u'txt': u'aaac'}])1061 assert_equals(1062 helpers.call_action(1063 'datastore_search',1064 resource_id=r['resource_id'],1065 records_format=u'csv',1066 sort=u'num, dt')['records'],1067 u'3,9,2020-01-01T00:00:00,aaac\n'1068 u'2,9,2020-01-02T00:00:00,aaab\n'1069 u'1,10,2020-01-01T00:00:00,aaab\n'1070 )1071 assert_equals(1072 helpers.call_action(1073 'datastore_search',1074 resource_id=r['resource_id'],1075 records_format=u'csv',1076 sort=u'dt, txt')['records'],1077 u'1,10,2020-01-01T00:00:00,aaab\n'1078 u'3,9,2020-01-01T00:00:00,aaac\n'1079 u'2,9,2020-01-02T00:00:00,aaab\n'1080 )1081 assert_equals(1082 helpers.call_action(1083 'datastore_search',1084 resource_id=r['resource_id'],1085 records_format=u'csv',1086 sort=u'txt, num')['records'],1087 u'2,9,2020-01-02T00:00:00,aaab\n'1088 u'1,10,2020-01-01T00:00:00,aaab\n'1089 u'3,9,2020-01-01T00:00:00,aaac\n'...

initData.js

Source:initData.js

1export const wsOverviewData={2 "default":{3 "type": "subscribe",4 "data": [5 6 {7 "resource_id": "overview",8 "details": {9 "data": "histogram",10 "callback": []11 }12 },13 {14 "resource_id": "put",15 "details": {16 "data": "aggregate_v2",17 "callback": []18 }19 },20 {21 "resource_id": "butlers",22 "details": {23 "data": "aggregate_v2",24 "callback": []25 }26 },27 {28 "resource_id": "chargers",29 "details": {30 "data": "aggregate_v2",31 "callback": []32 }33 },34 {35 "resource_id": "overview",36 "details": {37 "data": "pps_performance",38 "callback": []39 }40 },41 {42 "resource_id": "overview",43 "details": {44 "data": "pps_throughput",45 "callback": []46 }47 },48 49 {50 "resource_id": "audit",51 "details": {52 "data": "header",53 "callback": []54 }55 },56 {57 "resource_id": "overview",58 "details": {59 "data": "header",60 "callback": []61 }62 },63 {64 "resource_id": "zones",65 "details": {66 "data": "header",67 "callback": []68 }69 },70 {71 "resource_id": "users",72 "details": {73 "data": "header",74 "callback": []75 }76 },77 {78 "resource_id": "inventory_today",79 "details": {80 "data": "header",81 "callback": []82 }83 },84 {85 "resource_id": "status",86 "details": {87 "data": "header",88 "callback": []89 }90 },91 {92 "resource_id": "system",93 "details": {94 "data": "header",95 "callback": []96 }97 },98 {99 "resource_id": "audit_agg",100 "details": {101 "data": "aggregate_v2",102 "callback": []103 }104 }105 ,106 {107 "resource_id": "emergency",108 "details": {109 "data": "complete_v2",110 "callback": []111 }112 },113 {114 "resource_id": "orders",115 "details": {116 "data": "aggregate_v2",117 "callback": []118 }119 }120 ]121},122"users":{123 "type": "subscribe",124 "data": [125 {126 "resource_id": "users",127 "details": {128 "data": "complete_v2",129 "callback": []130 }131 },132 {133 "resource_id": "audit",134 "details": {135 "data": "header",136 "callback": []137 }138 },139 {140 "resource_id": "overview",141 "details": {142 "data": "header",143 "callback": []144 }145 },146 {147 "resource_id": "orders",148 "details": {149 "data": "header",150 "callback": []151 }152 },153 {154 "resource_id": "users",155 "details": {156 "data": "header",157 "callback": []158 }159 },160 {161 "resource_id": "inventory_today",162 "details": {163 "data": "header",164 "callback": []165 }166 },167 {168 "resource_id": "status",169 "details": {170 "data": "header",171 "callback": []172 }173 },174 {175 "resource_id": "system",176 "details": {177 "data": "header",178 "callback": []179 }180 }181 ,182 {183 "resource_id": "emergency",184 "details": {185 "data": "complete_v2",186 "callback": []187 }188 }189 ]190},191"system":{192 "type": "subscribe",193 "data": [194 {195 "resource_id": "butlers",196 "details": {197 "data": "complete_v2",198 "callback": []199 }200 },201 {202 "resource_id": "audit",203 "details": {204 "data": "header",205 "callback": []206 }207 },208 {209 "resource_id": "overview",210 "details": {211 "data": "header",212 "callback": []213 }214 },215 {216 "resource_id": "orders",217 "details": {218 "data": "header",219 "callback": []220 }221 },222 {223 "resource_id": "users",224 "details": {225 "data": "header",226 "callback": []227 }228 },229 {230 "resource_id": "inventory_today",231 "details": {232 "data": "header",233 "callback": []234 }235 },236 {237 "resource_id": "status",238 "details": {239 "data": "header",240 "callback": []241 }242 },243 {244 "resource_id": "system",245 "details": {246 "data": "header",247 "callback": []248 }249 }250 ,251 {252 "resource_id": "emergency",253 "details": {254 "data": "complete_v2",255 "callback": []256 }257 }258 ]259},260"pps":{261 "type": "subscribe",262 "data": [263 {264 "resource_id": "pps",265 "details": {266 "data": "complete_v2",267 "callback": []268 }269 },270 {271 "resource_id": "audit",272 "details": {273 "data": "header",274 "callback": []275 }276 },277 {278 "resource_id": "overview",279 "details": {280 "data": "header",281 "callback": []282 }283 },284 {285 "resource_id": "orders",286 "details": {287 "data": "header",288 "callback": []289 }290 },291 {292 "resource_id": "users",293 "details": {294 "data": "header",295 "callback": []296 }297 },298 {299 "resource_id": "inventory_today",300 "details": {301 "data": "header",302 "callback": []303 }304 },305 {306 "resource_id": "status",307 "details": {308 "data": "header",309 "callback": []310 }311 },312 {313 "resource_id": "system",314 "details": {315 "data": "header",316 "callback": []317 }318 }319 ,320 {321 "resource_id": "emergency",322 "details": {323 "data": "complete_v2",324 "callback": []325 }326 }327 ]328},329"chargingstation":{330 "type": "subscribe",331 "data":[332 {333 "resource_id": "chargers",334 "details": {335 "data": "complete_v2",336 "callback": []337 }338 },339 {340 "resource_id": "audit",341 "details": {342 "data": "header",343 "callback": []344 }345 },346 {347 "resource_id": "overview",348 "details": {349 "data": "header",350 "callback": []351 }352 },353 {354 "resource_id": "orders",355 "details": {356 "data": "header",357 "callback": []358 }359 },360 {361 "resource_id": "users",362 "details": {363 "data": "header",364 "callback": []365 }366 },367 {368 "resource_id": "inventory_today",369 "details": {370 "data": "header",371 "callback": []372 }373 },374 {375 "resource_id": "status",376 "details": {377 "data": "header",378 "callback": []379 }380 },381 {382 "resource_id": "system",383 "details": {384 "data": "header",385 "callback": []386 }387 }388 ,389 {390 "resource_id": "emergency",391 "details": {392 "data": "complete_v2",393 "callback": []394 }395 }396 ]397},398"notification":{399 "type": "subscribe",400 "data":[401 {402 "resource_id": "notification",403 "details": {404 "data": "complete_v2",405 "callback": []406 }407 },408 {409 "resource_id": "audit",410 "details": {411 "data": "header",412 "callback": []413 }414 },415 {416 "resource_id": "overview",417 "details": {418 "data": "header",419 "callback": []420 }421 },422 {423 "resource_id": "orders",424 "details": {425 "data": "header",426 "callback": []427 }428 },429 {430 "resource_id": "users",431 "details": {432 "data": "header",433 "callback": []434 }435 },436 {437 "resource_id": "inventory_today",438 "details": {439 "data": "header",440 "callback": []441 }442 },443 {444 "resource_id": "status",445 "details": {446 "data": "header",447 "callback": []448 }449 },450 {451 "resource_id": "system",452 "details": {453 "data": "header",454 "callback": []455 }456 }457 ,458 {459 "resource_id": "emergency",460 "details": {461 "data": "complete_v2",462 "callback": []463 }464 }465 ]466},467"butlerbots":{468 "type": "subscribe",469 "data":[470 {471 "resource_id": "butlers",472 "details": {473 "data": "complete_v2",474 "callback": []475 }476 },477 {478 "resource_id": "audit",479 "details": {480 "data": "header",481 "callback": []482 }483 },484 {485 "resource_id": "overview",486 "details": {487 "data": "header",488 "callback": []489 }490 },491 {492 "resource_id": "orders",493 "details": {494 "data": "header",495 "callback": []496 }497 },498 {499 "resource_id": "users",500 "details": {501 "data": "header",502 "callback": []503 }504 },505 {506 "resource_id": "inventory_today",507 "details": {508 "data": "header",509 "callback": []510 }511 },512 {513 "resource_id": "status",514 "details": {515 "data": "header",516 "callback": []517 }518 },519 {520 "resource_id": "system",521 "details": {522 "data": "header",523 "callback": []524 }525 }526 ,527 {528 "resource_id": "emergency",529 "details": {530 "data": "complete_v2",531 "callback": []532 }533 }534 ]535},536"inventory":{537 "type": "subscribe",538 "data": [539 540 {541 "resource_id": "inventory_today",542 "details": {543 "data": "complete_v2",544 "callback": []545 }546 },547 {548 "resource_id": "inventory_history",549 "details": {550 "data": "complete_v2",551 "callback": []552 }553 },554 {555 "resource_id": "audit",556 "details": {557 "data": "header",558 "callback": []559 }560 },561 {562 "resource_id": "overview",563 "details": {564 "data": "header",565 "callback": []566 }567 },568 {569 "resource_id": "orders",570 "details": {571 "data": "header",572 "callback": []573 }574 },575 {576 "resource_id": "users",577 "details": {578 "data": "header",579 "callback": []580 }581 },582 {583 "resource_id": "inventory_today",584 "details": {585 "data": "header",586 "callback": []587 }588 },589 {590 "resource_id": "status",591 "details": {592 "data": "header",593 "callback": []594 }595 },596 {597 "resource_id": "system",598 "details": {599 "data": "header",600 "callback": []601 }602 }603 ,604 {605 "resource_id": "emergency",606 "details": {607 "data": "complete_v2",608 "callback": []609 }610 }611 612 ]613},614"waves":{615 "type": "subscribe",616 "data": [617 {618 "resource_id": "waves",619 "details": {620 "data": "complete_v2",621 "callback": []622 }623 },624 {625 "resource_id": "audit",626 "details": {627 "data": "header",628 "callback": []629 }630 },631 {632 "resource_id": "overview",633 "details": {634 "data": "header",635 "callback": []636 }637 },638 {639 "resource_id": "orders",640 "details": {641 "data": "header",642 "callback": []643 }644 },645 {646 "resource_id": "users",647 "details": {648 "data": "header",649 "callback": []650 }651 },652 {653 "resource_id": "inventory_today",654 "details": {655 "data": "header",656 "callback": []657 }658 },659 {660 "resource_id": "status",661 "details": {662 "data": "header",663 "callback": []664 }665 },666 {667 "resource_id": "system",668 "details": {669 "data": "header",670 "callback": []671 }672 }673 ,674 {675 "resource_id": "emergency",676 "details": {677 "data": "complete_v2",678 "callback": []679 }680 }681 ]682},683"orders":{684 "type": "subscribe",685 "data": [686 {687 "resource_id": "orders",688 "details": {689 "data": "complete_v2",690 "callback": []691 }692 },693 {694 "resource_id": "audit",695 "details": {696 "data": "header",697 "callback": []698 }699 },700 {701 "resource_id": "overview",702 "details": {703 "data": "header",704 "callback": []705 }706 },707 {708 "resource_id": "waves",709 "details": {710 "data": "header",711 "callback": []712 }713 },714 {715 "resource_id": "users",716 "details": {717 "data": "header",718 "callback": []719 }720 },721 {722 "resource_id": "inventory_today",723 "details": {724 "data": "header",725 "callback": []726 }727 },728 {729 "resource_id": "status",730 "details": {731 "data": "header",732 "callback": []733 }734 },735 {736 "resource_id": "system",737 "details": {738 "data": "header",739 "callback": []740 }741 }742 ,743 {744 "resource_id": "emergency",745 "details": {746 "data": "complete_v2",747 "callback": []748 }749 }750 ]751},752"emergency":{753 "type": "subscribe",754 "data": [755 {756 "resource_id": "emergency",757 "details": {758 "data": "complete_v2",759 "callback": []760 }761 },762 {763 "resource_id": "audit",764 "details": {765 "data": "header",766 "callback": []767 }768 },769 {770 "resource_id": "overview",771 "details": {772 "data": "header",773 "callback": []774 }775 },776 {777 "resource_id": "waves",778 "details": {779 "data": "header",780 "callback": []781 }782 },783 {784 "resource_id": "users",785 "details": {786 "data": "header",787 "callback": []788 }789 },790 {791 "resource_id": "inventory_today",792 "details": {793 "data": "header",794 "callback": []795 }796 },797 {798 "resource_id": "status",799 "details": {800 "data": "header",801 "callback": []802 }803 },804 {805 "resource_id": "system",806 "details": {807 "data": "header",808 "callback": []809 }810 }811 ]812},813"zoning":{814 "type": "subscribe",815 "data": [816 {817 "resource_id": "zones",818 "details": {819 "data": "complete_v2",820 "callback": []821 }822 },823 {824 "resource_id": "zones",825 "details": {826 "data": "header",827 "callback": []828 }829 },830 {831 "resource_id": "emergency",832 "details": {833 "data": "complete_v2",834 "callback": []835 }836 },837 {838 "resource_id": "audit",839 "details": {840 "data": "header",841 "callback": []842 }843 },844 {845 "resource_id": "overview",846 "details": {847 "data": "header",848 "callback": []849 }850 },851 {852 "resource_id": "waves",853 "details": {854 "data": "header",855 "callback": []856 }857 },858 {859 "resource_id": "users",860 "details": {861 "data": "header",862 "callback": []863 }864 },865 {866 "resource_id": "inventory_today",867 "details": {868 "data": "header",869 "callback": []870 }871 },872 {873 "resource_id": "status",874 "details": {875 "data": "header",876 "callback": []877 }878 },879 {880 "resource_id": "system",881 "details": {882 "data": "header",883 "callback": []884 }885 }886 ]887},888"controllers": {889 "type": "subscribe",890 "data": [891 {892 "resource_id": "controllers",893 "details": {894 "data": "complete_v2",895 "callback": [],896 "filter_params": {897 }898 }899 },900 {901 "resource_id": "zones",902 "details": {903 "data": "header",904 "callback": []905 }906 },907 {908 "resource_id": "emergency",909 "details": {910 "data": "complete_v2",911 "callback": []912 }913 },914 {915 "resource_id": "audit",916 "details": {917 "data": "header",918 "callback": []919 }920 },921 {922 "resource_id": "overview",923 "details": {924 "data": "header",925 "callback": []926 }927 },928 {929 "resource_id": "waves",930 "details": {931 "data": "header",932 "callback": []933 }934 },935 {936 "resource_id": "users",937 "details": {938 "data": "header",939 "callback": []940 }941 },942 {943 "resource_id": "inventory_today",944 "details": {945 "data": "header",946 "callback": []947 }948 },949 {950 "resource_id": "status",951 "details": {952 "data": "header",953 "callback": []954 }955 },956 {957 "resource_id": "system",958 "details": {959 "data": "header",960 "callback": []961 }962 }963 ]964}...

test_upsert.py

Source:test_upsert.py

1# encoding: utf-82import json3import nose4import datetime5import sqlalchemy.orm as orm6import ckan.plugins as p7import ckan.lib.create_test_data as ctd8import ckan.model as model9import ckan.tests.legacy as tests10import ckan.tests.helpers as helpers11import ckan.tests.factories as factories12from ckan.common import config13import ckanext.datastore.backend.postgres as db14from ckanext.datastore.tests.helpers import (15 set_url_type, DatastoreFunctionalTestBase, DatastoreLegacyTestBase)16assert_equal = nose.tools.assert_equal17class TestDatastoreUpsertNewTests(DatastoreFunctionalTestBase):18 def test_upsert_doesnt_crash_with_json_field(self):19 resource = factories.Resource()20 data = {21 'resource_id': resource['id'],22 'force': True,23 'primary_key': 'id',24 'fields': [{'id': 'id', 'type': 'text'},25 {'id': 'book', 'type': 'json'},26 {'id': 'author', 'type': 'text'}],27 }28 helpers.call_action('datastore_create', **data)29 data = {30 'resource_id': resource['id'],31 'force': True,32 'method': 'insert',33 'records': [34 {'id': '1',35 'book': {'code': 'A', 'title': u'ñ'},36 'author': 'tolstoy'}],37 }38 helpers.call_action('datastore_upsert', **data)39 def test_upsert_doesnt_crash_with_json_field_with_string_value(self):40 resource = factories.Resource()41 data = {42 'resource_id': resource['id'],43 'force': True,44 'primary_key': 'id',45 'fields': [{'id': 'id', 'type': 'text'},46 {'id': 'book', 'type': 'json'},47 {'id': 'author', 'type': 'text'}],48 }49 helpers.call_action('datastore_create', **data)50 data = {51 'resource_id': resource['id'],52 'force': True,53 'method': 'insert',54 'records': [55 {'id': '1',56 'book': u'ñ',57 'author': 'tolstoy'}],58 }59 helpers.call_action('datastore_upsert', **data)60class TestDatastoreUpsert(DatastoreLegacyTestBase):61 sysadmin_user = None62 normal_user = None63 @classmethod64 def setup_class(cls):65 cls.app = helpers._get_test_app()66 super(TestDatastoreUpsert, cls).setup_class()67 ctd.CreateTestData.create()68 cls.sysadmin_user = model.User.get('testsysadmin')69 cls.normal_user = model.User.get('annafan')70 set_url_type(71 model.Package.get('annakarenina').resources, cls.sysadmin_user)72 resource = model.Package.get('annakarenina').resources[0]73 cls.data = {74 'resource_id': resource.id,75 'fields': [{'id': u'b\xfck', 'type': 'text'},76 {'id': 'author', 'type': 'text'},77 {'id': 'nested', 'type': 'json'},78 {'id': 'characters', 'type': 'text[]'},79 {'id': 'published'}],80 'primary_key': u'b\xfck',81 'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',82 'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]},83 {u'b\xfck': 'warandpeace', 'author': 'tolstoy',84 'nested': {'a':'b'}}85 ]86 }87 postparams = '%s=1' % json.dumps(cls.data)88 auth = {'Authorization': str(cls.sysadmin_user.apikey)}89 res = cls.app.post('/api/action/datastore_create', params=postparams,90 extra_environ=auth)91 res_dict = json.loads(res.body)92 assert res_dict['success'] is True93 engine = db.get_write_engine()94 cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))95 def test_upsert_requires_auth(self):96 data = {97 'resource_id': self.data['resource_id']98 }99 postparams = '%s=1' % json.dumps(data)100 res = self.app.post('/api/action/datastore_upsert', params=postparams,101 status=403)102 res_dict = json.loads(res.body)103 assert res_dict['success'] is False104 def test_upsert_empty_fails(self):105 postparams = '%s=1' % json.dumps({})106 auth = {'Authorization': str(self.sysadmin_user.apikey)}107 res = self.app.post('/api/action/datastore_upsert', params=postparams,108 extra_environ=auth, status=409)109 res_dict = json.loads(res.body)110 assert res_dict['success'] is False111 def test_upsert_basic(self):112 c = self.Session.connection()113 results = c.execute('select 1 from "{0}"'.format(self.data['resource_id']))114 assert results.rowcount == 2115 self.Session.remove()116 hhguide = u"hitchhiker's guide to the galaxy"117 data = {118 'resource_id': self.data['resource_id'],119 'method': 'upsert',120 'records': [{121 'author': 'adams',122 'nested': {'a': 2, 'b': {'c': 'd'}},123 'characters': ['Arthur Dent', 'Marvin'],124 'nested': {'foo': 'bar'},125 u'b\xfck': hhguide}]126 }127 postparams = '%s=1' % json.dumps(data)128 auth = {'Authorization': str(self.sysadmin_user.apikey)}129 res = self.app.post('/api/action/datastore_upsert', params=postparams,130 extra_environ=auth)131 res_dict = json.loads(res.body)132 assert res_dict['success'] is True133 c = self.Session.connection()134 results = c.execute('select * from "{0}"'.format(self.data['resource_id']))135 assert results.rowcount == 3136 records = results.fetchall()137 assert records[2][u'b\xfck'] == hhguide138 assert records[2].author == 'adams'139 assert records[2].characters == ['Arthur Dent', 'Marvin']140 assert json.loads(records[2].nested.json) == {'foo': 'bar'}141 self.Session.remove()142 c = self.Session.connection()143 results = c.execute("select * from \"{0}\" where author='{1}'".format(self.data['resource_id'], 'adams'))144 assert results.rowcount == 1145 self.Session.remove()146 # upsert only the publish date147 data = {148 'resource_id': self.data['resource_id'],149 'method': 'upsert',150 'records': [{'published': '1979-1-1', u'b\xfck': hhguide}]151 }152 postparams = '%s=1' % json.dumps(data)153 auth = {'Authorization': str(self.sysadmin_user.apikey)}154 res = self.app.post('/api/action/datastore_upsert', params=postparams,155 extra_environ=auth)156 res_dict = json.loads(res.body)157 assert res_dict['success'] is True158 c = self.Session.connection()159 results = c.execute('select * from "{0}"'.format(self.data['resource_id']))160 assert results.rowcount == 3161 records = results.fetchall()162 assert records[2][u'b\xfck'] == hhguide163 assert records[2].author == 'adams'164 assert records[2].published == datetime.datetime(1979, 1, 1)165 self.Session.remove()166 # delete publish date167 data = {168 'resource_id': self.data['resource_id'],169 'method': 'upsert',170 'records': [{u'b\xfck': hhguide, 'published': None}]171 }172 postparams = '%s=1' % json.dumps(data)173 auth = {'Authorization': str(self.sysadmin_user.apikey)}174 res = self.app.post('/api/action/datastore_upsert', params=postparams,175 extra_environ=auth)176 res_dict = json.loads(res.body)177 assert res_dict['success'] is True178 c = self.Session.connection()179 results = c.execute('select * from "{0}"'.format(self.data['resource_id']))180 assert results.rowcount == 3181 records = results.fetchall()182 assert records[2][u'b\xfck'] == hhguide183 assert records[2].author == 'adams'184 assert records[2].published == None185 self.Session.remove()186 data = {187 'resource_id': self.data['resource_id'],188 'method': 'upsert',189 'records': [{'author': 'tolkien', u'b\xfck': 'the hobbit'}]190 }191 postparams = '%s=1' % json.dumps(data)192 auth = {'Authorization': str(self.sysadmin_user.apikey)}193 res = self.app.post('/api/action/datastore_upsert', params=postparams,194 extra_environ=auth)195 res_dict = json.loads(res.body)196 assert res_dict['success'] is True197 c = self.Session.connection()198 results = c.execute('select * from "{0}"'.format(self.data['resource_id']))199 assert results.rowcount == 4200 records = results.fetchall()201 assert records[3][u'b\xfck'] == 'the hobbit'202 assert records[3].author == 'tolkien'203 self.Session.remove()204 # test % in records205 data = {206 'resource_id': self.data['resource_id'],207 'method': 'upsert',208 'records': [{'author': 'tol % kien', u'b\xfck': 'the % hobbit'}]209 }210 postparams = '%s=1' % json.dumps(data)211 auth = {'Authorization': str(self.sysadmin_user.apikey)}212 res = self.app.post('/api/action/datastore_upsert', params=postparams,213 extra_environ=auth)214 res_dict = json.loads(res.body)215 assert res_dict['success'] is True216 def test_upsert_missing_key(self):217 data = {218 'resource_id': self.data['resource_id'],219 'method': 'upsert',220 'records': [{'author': 'tolkien'}]221 }222 postparams = '%s=1' % json.dumps(data)223 auth = {'Authorization': str(self.sysadmin_user.apikey)}224 res = self.app.post('/api/action/datastore_upsert', params=postparams,225 extra_environ=auth, status=409)226 res_dict = json.loads(res.body)227 assert res_dict['success'] is False228 def test_upsert_non_existing_field(self):229 data = {230 'resource_id': self.data['resource_id'],231 'method': 'upsert',232 'records': [{u'b\xfck': 'annakarenina', 'dummy': 'tolkien'}]233 }234 postparams = '%s=1' % json.dumps(data)235 auth = {'Authorization': str(self.sysadmin_user.apikey)}236 res = self.app.post('/api/action/datastore_upsert', params=postparams,237 extra_environ=auth, status=409)238 res_dict = json.loads(res.body)239 assert res_dict['success'] is False240 def test_upsert_works_with_empty_list_in_json_field(self):241 hhguide = u"hitchhiker's guide to the galaxy"242 data = {243 'resource_id': self.data['resource_id'],244 'method': 'upsert',245 'records': [{246 'nested': [],247 u'b\xfck': hhguide}]248 }249 postparams = '%s=1' % json.dumps(data)250 auth = {'Authorization': str(self.sysadmin_user.apikey)}251 res = self.app.post('/api/action/datastore_upsert', params=postparams,252 extra_environ=auth)253 res_dict = json.loads(res.body)254 assert res_dict['success'] is True, res_dict255 c = self.Session.connection()256 results = c.execute('select * from "{0}"'.format(data['resource_id']))257 record = [r for r in results.fetchall() if r[2] == hhguide]258 self.Session.remove()259 assert len(record) == 1, record260 assert_equal(json.loads(record[0][4].json),261 data['records'][0]['nested'])262class TestDatastoreInsert(DatastoreLegacyTestBase):263 sysadmin_user = None264 normal_user = None265 @classmethod266 def setup_class(cls):267 cls.app = helpers._get_test_app()268 super(TestDatastoreInsert, cls).setup_class()269 ctd.CreateTestData.create()270 cls.sysadmin_user = model.User.get('testsysadmin')271 cls.normal_user = model.User.get('annafan')272 set_url_type(273 model.Package.get('annakarenina').resources, cls.sysadmin_user)274 resource = model.Package.get('annakarenina').resources[0]275 cls.data = {276 'resource_id': resource.id,277 'fields': [{'id': u'b\xfck', 'type': 'text'},278 {'id': 'author', 'type': 'text'},279 {'id': 'nested', 'type': 'json'},280 {'id': 'characters', 'type': 'text[]'},281 {'id': 'published'}],282 'primary_key': u'b\xfck',283 'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',284 'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]},285 {u'b\xfck': 'warandpeace', 'author': 'tolstoy',286 'nested': {'a':'b'}}287 ]288 }289 postparams = '%s=1' % json.dumps(cls.data)290 auth = {'Authorization': str(cls.sysadmin_user.apikey)}291 res = cls.app.post('/api/action/datastore_create', params=postparams,292 extra_environ=auth)293 res_dict = json.loads(res.body)294 assert res_dict['success'] is True295 engine = db.get_write_engine()296 cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))297 def test_insert_non_existing_field(self):298 data = {299 'resource_id': self.data['resource_id'],300 'method': 'insert',301 'records': [{u'b\xfck': 'the hobbit', 'dummy': 'tolkien'}]302 }303 postparams = '%s=1' % json.dumps(data)304 auth = {'Authorization': str(self.sysadmin_user.apikey)}305 res = self.app.post('/api/action/datastore_upsert', params=postparams,306 extra_environ=auth, status=409)307 res_dict = json.loads(res.body)308 assert res_dict['success'] is False309 def test_insert_with_index_violation(self):310 data = {311 'resource_id': self.data['resource_id'],312 'method': 'insert',313 'records': [{u'b\xfck': 'annakarenina'}]314 }315 postparams = '%s=1' % json.dumps(data)316 auth = {'Authorization': str(self.sysadmin_user.apikey)}317 res = self.app.post('/api/action/datastore_upsert', params=postparams,318 extra_environ=auth, status=409)319 res_dict = json.loads(res.body)320 assert res_dict['success'] is False321 def test_insert_basic(self):322 hhguide = u"hitchhiker's guide to the galaxy"323 data = {324 'resource_id': self.data['resource_id'],325 'method': 'insert',326 'records': [{327 'author': 'adams',328 'characters': ['Arthur Dent', 'Marvin'],329 'nested': {'foo': 'bar', 'baz': 3},330 u'b\xfck': hhguide}]331 }332 postparams = '%s=1' % json.dumps(data)333 auth = {'Authorization': str(self.sysadmin_user.apikey)}334 res = self.app.post('/api/action/datastore_upsert', params=postparams,335 extra_environ=auth)336 res_dict = json.loads(res.body)337 assert res_dict['success'] is True338 c = self.Session.connection()339 results = c.execute('select * from "{0}"'.format(self.data['resource_id']))340 self.Session.remove()341 assert results.rowcount == 3342class TestDatastoreUpdate(DatastoreLegacyTestBase):343 sysadmin_user = None344 normal_user = None345 @classmethod346 def setup_class(cls):347 cls.app = helpers._get_test_app()348 super(TestDatastoreUpdate, cls).setup_class()349 ctd.CreateTestData.create()350 cls.sysadmin_user = model.User.get('testsysadmin')351 cls.normal_user = model.User.get('annafan')352 set_url_type(353 model.Package.get('annakarenina').resources, cls.sysadmin_user)354 resource = model.Package.get('annakarenina').resources[0]355 hhguide = u"hitchhiker's guide to the galaxy"356 cls.data = {357 'resource_id': resource.id,358 'fields': [{'id': u'b\xfck', 'type': 'text'},359 {'id': 'author', 'type': 'text'},360 {'id': 'nested', 'type': 'json'},361 {'id': 'characters', 'type': 'text[]'},362 {'id': 'published'}],363 'primary_key': u'b\xfck',364 'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',365 'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]},366 {u'b\xfck': 'warandpeace', 'author': 'tolstoy',367 'nested': {'a':'b'}},368 {'author': 'adams',369 'characters': ['Arthur Dent', 'Marvin'],370 'nested': {'foo': 'bar'},371 u'b\xfck': hhguide}372 ]373 }374 postparams = '%s=1' % json.dumps(cls.data)375 auth = {'Authorization': str(cls.sysadmin_user.apikey)}376 res = cls.app.post('/api/action/datastore_create', params=postparams,377 extra_environ=auth)378 res_dict = json.loads(res.body)379 assert res_dict['success'] is True380 engine = db.get_write_engine()381 cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))382 def test_update_basic(self):383 c = self.Session.connection()384 results = c.execute('select 1 from "{0}"'.format(self.data['resource_id']))385 assert results.rowcount == 3, results.rowcount386 self.Session.remove()387 hhguide = u"hitchhiker's guide to the galaxy"388 data = {389 'resource_id': self.data['resource_id'],390 'method': 'update',391 'records': [{392 'author': 'adams',393 'characters': ['Arthur Dent', 'Marvin'],394 'nested': {'baz': 3},395 u'b\xfck': hhguide}]396 }397 postparams = '%s=1' % json.dumps(data)398 auth = {'Authorization': str(self.sysadmin_user.apikey)}399 res = self.app.post('/api/action/datastore_upsert', params=postparams,400 extra_environ=auth)401 res_dict = json.loads(res.body)402 assert res_dict['success'] is True403 c = self.Session.connection()404 results = c.execute('select * from "{0}"'.format(self.data['resource_id']))405 assert results.rowcount == 3406 records = results.fetchall()407 assert records[2][u'b\xfck'] == hhguide408 assert records[2].author == 'adams'409 self.Session.remove()410 c = self.Session.connection()411 results = c.execute("select * from \"{0}\" where author='{1}'".format(self.data['resource_id'], 'adams'))412 assert results.rowcount == 1413 self.Session.remove()414 # update only the publish date415 data = {416 'resource_id': self.data['resource_id'],417 'method': 'update',418 'records': [{'published': '1979-1-1', u'b\xfck': hhguide}]419 }420 postparams = '%s=1' % json.dumps(data)421 auth = {'Authorization': str(self.sysadmin_user.apikey)}422 res = self.app.post('/api/action/datastore_upsert', params=postparams,423 extra_environ=auth)424 res_dict = json.loads(res.body)425 assert res_dict['success'] is True426 c = self.Session.connection()427 results = c.execute('select * from "{0}"'.format(self.data['resource_id']))428 self.Session.remove()429 assert results.rowcount == 3430 records = results.fetchall()431 assert records[2][u'b\xfck'] == hhguide432 assert records[2].author == 'adams'433 assert records[2].published == datetime.datetime(1979, 1, 1)434 # delete publish date435 data = {436 'resource_id': self.data['resource_id'],437 'method': 'update',438 'records': [{u'b\xfck': hhguide, 'published': None}]439 }440 postparams = '%s=1' % json.dumps(data)441 auth = {'Authorization': str(self.sysadmin_user.apikey)}442 res = self.app.post('/api/action/datastore_upsert', params=postparams,443 extra_environ=auth)444 res_dict = json.loads(res.body)445 assert res_dict['success'] is True446 c = self.Session.connection()447 results = c.execute('select * from "{0}"'.format(self.data['resource_id']))448 self.Session.remove()449 assert results.rowcount == 3450 records = results.fetchall()451 assert records[2][u'b\xfck'] == hhguide452 assert records[2].author == 'adams'453 assert records[2].published == None454 def test_update_missing_key(self):455 data = {456 'resource_id': self.data['resource_id'],457 'method': 'update',458 'records': [{'author': 'tolkien'}]459 }460 postparams = '%s=1' % json.dumps(data)461 auth = {'Authorization': str(self.sysadmin_user.apikey)}462 res = self.app.post('/api/action/datastore_upsert', params=postparams,463 extra_environ=auth, status=409)464 res_dict = json.loads(res.body)465 assert res_dict['success'] is False466 def test_update_non_existing_key(self):467 data = {468 'resource_id': self.data['resource_id'],469 'method': 'update',470 'records': [{u'b\xfck': '', 'author': 'tolkien'}]471 }472 postparams = '%s=1' % json.dumps(data)473 auth = {'Authorization': str(self.sysadmin_user.apikey)}474 res = self.app.post('/api/action/datastore_upsert', params=postparams,475 extra_environ=auth, status=409)476 res_dict = json.loads(res.body)477 assert res_dict['success'] is False478 def test_update_non_existing_field(self):479 data = {480 'resource_id': self.data['resource_id'],481 'method': 'update',482 'records': [{u'b\xfck': 'annakarenina', 'dummy': 'tolkien'}]483 }484 postparams = '%s=1' % json.dumps(data)485 auth = {'Authorization': str(self.sysadmin_user.apikey)}486 res = self.app.post('/api/action/datastore_upsert', params=postparams,487 extra_environ=auth, status=409)488 res_dict = json.loads(res.body)...

competencies_resources.js

Source:competencies_resources.js

1module.exports = {2 tableName: "competencies_competencies_resources",3 records: [4 {5 competency_id: 1,6 resource_id: 17 },8 {9 competency_id: 1,10 resource_id: 211 },12 {13 competency_id: 2,14 resource_id: 315 },16 {17 competency_id: 3,18 resource_id: 419 },20 {21 competency_id: 4,22 resource_id: 423 },24 {25 competency_id: 4,26 resource_id: 527 },28 {29 competency_id: 5,30 resource_id: 431 },32 {33 competency_id: 5,34 resource_id: 635 },36 {37 competency_id: 5,38 resource_id: 739 },40 {41 competency_id: 5,42 resource_id: 843 },44 {45 competency_id: 6,46 resource_id: 947 },48 {49 competency_id: 7,50 resource_id: 851 },52 {53 competency_id: 8,54 resource_id: 855 },56 {57 competency_id: 8,58 resource_id: 1059 },60 {61 competency_id: 8,62 resource_id: 1163 },64 {65 competency_id: 9,66 resource_id: 967 },68 {69 competency_id: 10,70 resource_id: 1271 },72 {73 competency_id: 10,74 resource_id: 1375 },76 {77 competency_id: 11,78 resource_id: 979 },80 {81 competency_id: 12,82 resource_id: 983 },84 {85 competency_id: 13,86 resource_id: 987 },88 {89 competency_id: 14,90 resource_id: 1491 },92 {93 competency_id: 14,94 resource_id: 1595 },96 {97 competency_id: 14,98 resource_id: 1699 },100 {101 competency_id: 15,102 resource_id: 17103 },104 {105 competency_id: 15,106 resource_id: 18107 },108 {109 competency_id: 15,110 resource_id: 18111 },112 {113 competency_id: 15,114 resource_id: 19115 },116 {117 competency_id: 15,118 resource_id: 20119 },120 {121 competency_id: 16,122 resource_id: 21123 },124 {125 competency_id: 16,126 resource_id: 22127 },128 {129 competency_id: 16,130 resource_id: 23131 },132 {133 competency_id: 17,134 resource_id: 23135 },136 {137 competency_id: 17,138 resource_id: 24139 },140 {141 competency_id: 17,142 resource_id: 25143 },144 {145 competency_id: 17,146 resource_id: 26147 },148 {149 competency_id: 17,150 resource_id: 27151 },152 {153 competency_id: 18,154 resource_id: 9155 },156 {157 competency_id: 19,158 resource_id: 9159 },160 {161 competency_id: 20,162 resource_id: 28163 },164 {165 competency_id: 20,166 resource_id: 26167 },168 {169 competency_id: 21,170 resource_id: 26171 },172 {173 competency_id: 22,174 resource_id: 29175 },176 {177 competency_id: 22,178 resource_id: 30179 },180 {181 competency_id: 22,182 resource_id: 31183 },184 {185 competency_id: 22,186 resource_id: 32187 },188 {189 competency_id: 22,190 resource_id: 33191 },192 {193 competency_id: 22,194 resource_id: 25195 },196 {197 competency_id: 23,198 resource_id: 34199 },200 {201 competency_id: 24,202 resource_id: 34203 },204 {205 competency_id: 25,206 resource_id: 35207 },208 {209 competency_id: 25,210 resource_id: 36211 },212 {213 competency_id: 25,214 resource_id: 37215 },216 {217 competency_id: 25,218 resource_id: 33219 },220 {221 competency_id: 25,222 resource_id: 25223 },224 {225 competency_id: 26,226 resource_id: 38227 },228 {229 competency_id: 26,230 resource_id: 39231 },232 {233 competency_id: 27,234 resource_id: 40235 },236 {237 competency_id: 28,238 resource_id: 25239 },240 {241 competency_id: 28,242 resource_id: 41243 },244 {245 competency_id: 29,246 resource_id: 42247 },248 {249 competency_id: 29,250 resource_id: 25251 },252 {253 competency_id: 30,254 resource_id: 43255 },256 {257 competency_id: 30,258 resource_id: 44259 },260 {261 competency_id: 30,262 resource_id: 25263 },264 {265 competency_id: 30,266 resource_id: 26267 },268 {269 competency_id: 31,270 resource_id: 45271 },272 {273 competency_id: 31,274 resource_id: 26275 },276 {277 competency_id: 32,278 resource_id: 25279 },280 {281 competency_id: 32,282 resource_id: 26283 },284 {285 competency_id: 33,286 resource_id: 46287 },288 {289 competency_id: 33,290 resource_id: 26291 },292 {293 competency_id: 34,294 resource_id: 25295 },296 {297 competency_id: 34,298 resource_id: 26299 },300 {301 competency_id: 35,302 resource_id: 46303 },304 {305 competency_id: 35,306 resource_id: 26307 },308 {309 competency_id: 35,310 resource_id: 47311 },312 {313 competency_id: 36,314 resource_id: 48315 },316 {317 competency_id: 36,318 resource_id: 25319 },320 {321 competency_id: 37,322 resource_id: 48323 },324 {325 competency_id: 37,326 resource_id: 25327 },328 {329 competency_id: 38,330 resource_id: 49331 },332 {333 competency_id: 38,334 resource_id: 50335 },336 {337 competency_id: 39,338 resource_id: 50339 },340 {341 competency_id: 40,342 resource_id: 26343 },344 {345 competency_id: 40,346 resource_id: 51347 },348 {349 competency_id: 41,350 resource_id: 52351 },352 {353 competency_id: 41,354 resource_id: 53355 },356 {357 competency_id: 41,358 resource_id: 54359 },360 {361 competency_id: 41,362 resource_id: 25363 },364 {365 competency_id: 41,366 resource_id: 55367 },368 {369 competency_id: 41,370 resource_id: 56371 },372 {373 competency_id: 41,374 resource_id: 51375 },376 {377 competency_id: 41,378 resource_id: 57379 },380 {381 competency_id: 42,382 resource_id: 53383 },384 {385 competency_id: 42,386 resource_id: 56387 },388 {389 competency_id: 43,390 resource_id: 58391 },392 {393 competency_id: 44,394 resource_id: 27395 },396 {397 competency_id: 45,398 resource_id: 25399 },400 {401 competency_id: 45,402 resource_id: 26403 },404 {405 competency_id: 45,406 resource_id: 59407 },408 {409 competency_id: 45,410 resource_id: 60411 },412 {413 competency_id: 46,414 resource_id: 46415 },416 {417 competency_id: 47,418 resource_id: 26419 },420 {421 competency_id: 47,422 resource_id: 27423 },424 {425 competency_id: 47,426 resource_id: 39427 },428 {429 competency_id: 48,430 resource_id: 46431 },432 {433 competency_id: 49,434 resource_id: 25435 },436 {437 competency_id: 50,438 resource_id: 25439 },440 {441 competency_id: 50,442 resource_id: 26443 },444 {445 competency_id: 51,446 resource_id: 26447 },448 {449 competency_id: 52,450 resource_id: 25451 },452 {453 competency_id: 52,454 resource_id: 26455 },456 {457 competency_id: 53,458 resource_id: 9459 },460 {461 competency_id: 54,462 resource_id: 9463 },464 {465 competency_id: 54,466 resource_id: 61467 },468 {469 competency_id: 54,470 resource_id: 62471 },472 {473 competency_id: 54,474 resource_id: 25475 },476 {477 competency_id: 54,478 resource_id: 26479 },480 {481 competency_id: 55,482 resource_id: 25483 },484 {485 competency_id: 55,486 resource_id: 26487 },488 {489 competency_id: 55,490 resource_id: 39491 },492 {493 competency_id: 56,494 resource_id: 63495 },496 {497 competency_id: 56,498 resource_id: 61499 },500 {501 competency_id: 56,502 resource_id: 42503 },504 {505 competency_id: 56,506 resource_id: 26507 },508 {509 competency_id: 56,510 resource_id: 39511 },512 {513 competency_id: 55,514 resource_id: 64515 },516 {517 competency_id: 57,518 resource_id: 26519 },520 {521 competency_id: 58,522 resource_id: 25523 },524 {525 competency_id: 58,526 resource_id: 26527 },528 {529 competency_id: 59,530 resource_id: 65531 },532 {533 competency_id: 60,534 resource_id: 66535 },536 {537 competency_id: 60,538 resource_id: 67539 },540 {541 competency_id: 60,542 resource_id: 25543 },544 {545 competency_id: 60,546 resource_id: 26547 },548 {549 competency_id: 60,550 resource_id: 39551 },552 {553 competency_id: 61,554 resource_id: 66555 },556 {557 competency_id: 61,558 resource_id: 26559 },560 {561 competency_id: 62,562 resource_id: 68563 },564 {565 competency_id: 62,566 resource_id: 26567 },568 {569 competency_id: 63,570 resource_id: 9571 },572 {573 competency_id: 63,574 resource_id: 69575 },576 {577 competency_id: 63,578 resource_id: 25579 },580 {581 competency_id: 63,582 resource_id: 26583 },584 {585 competency_id: 64,586 resource_id: 69587 },588 {589 competency_id: 65,590 resource_id: 69591 },592 {593 competency_id: 65,594 resource_id: 25595 },596 {597 competency_id: 66,598 resource_id: 9599 },600 {601 competency_id: 67,602 resource_id: 70603 },604 {605 competency_id: 68,606 resource_id: 70607 },608 {609 competency_id: 68,610 resource_id: 25611 },612 {613 competency_id: 69,614 resource_id: 70615 },616 {617 competency_id: 70,618 resource_id: 70619 },620 {621 competency_id: 71,622 resource_id: 70623 },624 {625 competency_id: 72,626 resource_id: 71627 },628 {629 competency_id: 73,630 resource_id: 9631 },632 {633 competency_id: 74,634 resource_id: 72635 },636 {637 competency_id: 75,638 resource_id: 73639 },640 {641 competency_id: 76,642 resource_id: 25643 },644 {645 competency_id: 77,646 resource_id: 9647 },648 {649 competency_id: 78,650 resource_id: 9651 },652 {653 competency_id: 79,654 resource_id: 9655 },656 {657 competency_id: 80,658 resource_id: 9659 },660 {661 competency_id: 81,662 resource_id: 74663 },664 {665 competency_id: 81,666 resource_id: 25667 },668 {669 competency_id: 82,670 resource_id: 74671 },672 {673 competency_id: 82,674 resource_id: 68675 },676 {677 competency_id: 82,678 resource_id: 25679 },680 {681 competency_id: 82,682 resource_id: 62683 },684 {685 competency_id: 83,686 resource_id: 75687 },688 {689 competency_id: 83,690 resource_id: 74691 },692 {693 competency_id: 84,694 resource_id: 25695 },696 {697 competency_id: 84,698 resource_id: 76699 },700 {701 competency_id: 85,702 resource_id: 77703 },704 {705 competency_id: 86,706 resource_id: 25707 },708 {709 competency_id: 87,710 resource_id: 62711 },712 {713 competency_id: 87,714 resource_id: 25715 },716 {717 competency_id: 88,718 resource_id: 22719 },720 {721 competency_id: 88,722 resource_id: 78723 },724 {725 competency_id: 13,726 resource_id: 9727 },728 {729 competency_id: 89,730 resource_id: 22731 },732 {733 competency_id: 89,734 resource_id: 79735 },736 {737 competency_id: 90,738 resource_id: 79739 },740 {741 competency_id: 13,742 resource_id: 9743 },744 {745 competency_id: 91,746 resource_id: 65747 },748 {749 competency_id: 92,750 resource_id: 9751 },752 {753 competency_id: 93,754 resource_id: 9755 }756 ]...

test_db.py

Source:test_db.py

1# encoding: utf-82import mock3import nose4import sqlalchemy.exc5import ckan.plugins as p6import ckan.lib.jobs as jobs7import ckan.tests.helpers as helpers8import ckan.tests.factories as factories9import ckanext.datastore.backend.postgres as db10import ckanext.datastore.backend as backend11from ckanext.datastore.tests.helpers import DatastoreFunctionalTestBase12assert_equal = nose.tools.assert_equal13class TestCreateIndexes(object):14 @helpers.change_config('ckan.datastore.default_fts_index_method', None)15 def test_creates_fts_index_using_gist_by_default(self):16 connection = mock.MagicMock()17 context = {18 'connection': connection19 }20 resource_id = 'resource_id'21 data_dict = {22 'resource_id': resource_id,23 }24 db.create_indexes(context, data_dict)25 self._assert_created_index_on('_full_text', connection, resource_id,26 method='gist')27 @helpers.change_config('ckan.datastore.default_fts_index_method', 'gin')28 def test_default_fts_index_method_can_be_overwritten_by_config_var(self):29 connection = mock.MagicMock()30 context = {31 'connection': connection32 }33 resource_id = 'resource_id'34 data_dict = {35 'resource_id': resource_id,36 }37 db.create_indexes(context, data_dict)38 self._assert_created_index_on('_full_text', connection, resource_id,39 method='gin')40 @helpers.change_config('ckan.datastore.default_fts_lang', None)41 @mock.patch('ckanext.datastore.backend.postgres._get_fields')42 def test_creates_fts_index_on_all_fields_except_dates_nested_and_arrays_with_english_as_default(self, _get_fields):43 _get_fields.return_value = [44 {'id': 'text', 'type': 'text'},45 {'id': 'number', 'type': 'number'},46 {'id': 'nested', 'type': 'nested'},47 {'id': 'date', 'type': 'date'},48 {'id': 'text array', 'type': 'text[]'},49 {'id': 'timestamp', 'type': 'timestamp'},50 ]51 connection = mock.MagicMock()52 context = {53 'connection': connection54 }55 resource_id = 'resource_id'56 data_dict = {57 'resource_id': resource_id,58 }59 db.create_indexes(context, data_dict)60 self._assert_created_index_on('text', connection, resource_id, 'english')61 self._assert_created_index_on('number', connection, resource_id, 'english', cast=True)62 @helpers.change_config('ckan.datastore.default_fts_lang', 'simple')63 @mock.patch('ckanext.datastore.backend.postgres._get_fields')64 def test_creates_fts_index_on_textual_fields_can_overwrite_lang_with_config_var(self, _get_fields):65 _get_fields.return_value = [66 {'id': 'foo', 'type': 'text'},67 ]68 connection = mock.MagicMock()69 context = {70 'connection': connection71 }72 resource_id = 'resource_id'73 data_dict = {74 'resource_id': resource_id,75 }76 db.create_indexes(context, data_dict)77 self._assert_created_index_on('foo', connection, resource_id, 'simple')78 @helpers.change_config('ckan.datastore.default_fts_lang', 'simple')79 @mock.patch('ckanext.datastore.backend.postgres._get_fields')80 def test_creates_fts_index_on_textual_fields_can_overwrite_lang_using_lang_param(self, _get_fields):81 _get_fields.return_value = [82 {'id': 'foo', 'type': 'text'},83 ]84 connection = mock.MagicMock()85 context = {86 'connection': connection87 }88 resource_id = 'resource_id'89 data_dict = {90 'resource_id': resource_id,91 'lang': 'french',92 }93 db.create_indexes(context, data_dict)94 self._assert_created_index_on('foo', connection, resource_id, 'french')95 def _assert_created_index_on(self, field, connection, resource_id,96 lang=None, cast=False, method='gist'):97 field = u'"{0}"'.format(field)98 if cast:99 field = u'cast({0} AS text)'.format(field)100 if lang is not None:101 sql_str = (u'ON "resource_id" '102 u'USING {method}(to_tsvector(\'{lang}\', {field}))')103 sql_str = sql_str.format(method=method, lang=lang, field=field)104 else:105 sql_str = u'USING {method}({field})'.format(method=method,106 field=field)107 calls = connection.execute.call_args_list108 was_called = [call for call in calls if call[0][0].find(sql_str) != -1]109 assert was_called, ("Expected 'connection.execute' to have been "110 "called with a string containing '%s'" % sql_str)111@mock.patch("ckanext.datastore.backend.postgres._get_fields")112def test_upsert_with_insert_method_and_invalid_data(113 mock_get_fields_function):114 """upsert_data() should raise InvalidDataError if given invalid data.115 If the type of a field is numeric and upsert_data() is given a whitespace116 value like " ", it should raise DataError.117 In this case we're testing with "method": "insert" in the data_dict.118 """119 mock_connection = mock.Mock()120 mock_connection.execute.side_effect = sqlalchemy.exc.DataError(121 "statement", "params", "orig", connection_invalidated=False)122 context = {123 "connection": mock_connection,124 }125 data_dict = {126 "fields": [{"id": "value", "type": "numeric"}],127 "records": [128 {"value": 0},129 {"value": 1},130 {"value": 2},131 {"value": 3},132 {"value": " "}, # Invalid numeric value.133 {"value": 5},134 {"value": 6},135 {"value": 7},136 ],137 "method": "insert",138 "resource_id": "fake-resource-id",139 }140 mock_get_fields_function.return_value = data_dict["fields"]141 nose.tools.assert_raises(142 backend.InvalidDataError, db.upsert_data, context, data_dict)143class TestGetAllResourcesIdsInDatastore(DatastoreFunctionalTestBase):144 def test_get_all_resources_ids_in_datastore(self):145 resource_in_datastore = factories.Resource()146 resource_not_in_datastore = factories.Resource()147 data = {148 'resource_id': resource_in_datastore['id'],149 'force': True,150 }151 helpers.call_action('datastore_create', **data)152 resource_ids = backend.get_all_resources_ids_in_datastore()153 assert resource_in_datastore['id'] in resource_ids154 assert resource_not_in_datastore['id'] not in resource_ids155def datastore_job(res_id, value):156 '''157 A background job that uses the Datastore.158 '''159 app = helpers._get_test_app()160 p.load('datastore')161 data = {162 'resource_id': res_id,163 'method': 'insert',164 'records': [{'value': value}],165 }166 with app.flask_app.test_request_context():167 helpers.call_action('datastore_upsert', **data)168class TestBackgroundJobs(helpers.RQTestBase, DatastoreFunctionalTestBase):169 '''170 Test correct interaction with the background jobs system.171 '''172 def test_worker_datastore_access(self):173 '''174 Test DataStore access from within a worker.175 '''176 pkg = factories.Dataset()177 data = {178 'resource': {179 'package_id': pkg['id'],180 },181 'fields': [{'id': 'value', 'type': 'int'}],182 }183 with self._get_test_app().flask_app.test_request_context():184 table = helpers.call_action('datastore_create', **data)185 res_id = table['resource_id']186 for i in range(3):187 self.enqueue(datastore_job, args=[res_id, i])188 jobs.Worker().work(burst=True)189 # Aside from ensuring that the job succeeded, this also checks190 # that accessing the Datastore still works in the main process.191 result = helpers.call_action('datastore_search', resource_id=res_id)...

price-monitor-stack.ts

Source:price-monitor-stack.ts

1import * as path from "path";2import { Construct } from "constructs";3import {4 aws_events as events,5 aws_events_targets as targets,6 aws_lambda_nodejs as lambdaNodejs,7 aws_lambda as lambda,8 aws_dynamodb as dynamodb,9 aws_iam as iam,10 aws_lambda_event_sources as awsLambdaEventSources,11 aws_cloudwatch as cloudwatch,12 aws_cloudwatch_actions as cloudwatchActions,13 aws_sns as sns,14 aws_sns_subscriptions as snsSubscriptions,15 aws_ssm as ssm,16 Stack,17 StackProps,18 Duration,19} from "aws-cdk-lib";20const RESOURCE_ID = {21 DYNAMODB_TABLE_PRICES: "DynamoDbTablePrices",22 SSM_PARAMETER_EMAIL_ALERTS: "SsmParameterEmailAlerts",23 SNS_TOPIC_ALERTS: "SnsTopicAlerts",24 LAMBDA_PRICE_CHECK: "LambdaPriceCheck",25 LAMBDA_NOTOFICATION: "LambdaNotification",26 LAMBDA_PRICE_CHECK_ALARM: "LambdaPriceCheckAlarm",27 LAMBDA_NOTOFICATION_ALARM: "LambdaNotificationAlarm",28 SCHEDULED_EVENT: "ScheduledEvent",29};30export class PriceMonitorStack extends Stack {31 constructor(scope: Construct, id: string, props?: StackProps) {32 super(scope, id, props);33 const dynamoDbTablePrices = new dynamodb.Table(34 this,35 RESOURCE_ID.DYNAMODB_TABLE_PRICES,36 {37 tableName: `${id}-${RESOURCE_ID.DYNAMODB_TABLE_PRICES}`,38 partitionKey: {39 name: "id",40 type: dynamodb.AttributeType.STRING,41 },42 stream: dynamodb.StreamViewType.NEW_AND_OLD_IMAGES,43 }44 );45 const ssmParameterEmailAlerts =46 ssm.StringParameter.fromStringParameterAttributes(47 this,48 RESOURCE_ID.SSM_PARAMETER_EMAIL_ALERTS,49 {50 parameterName: `/${id}/EmailAlerts`,51 }52 ).stringValue;53 const snsTopicAlerts = new sns.Topic(this, RESOURCE_ID.SNS_TOPIC_ALERTS, {54 topicName: `${id}-${RESOURCE_ID.SNS_TOPIC_ALERTS}`,55 });56 snsTopicAlerts.addSubscription(57 new snsSubscriptions.EmailSubscription(ssmParameterEmailAlerts)58 );59 const lambdaPriceCheck = new lambdaNodejs.NodejsFunction(60 this,61 RESOURCE_ID.LAMBDA_PRICE_CHECK,62 {63 functionName: `${id}-${RESOURCE_ID.LAMBDA_PRICE_CHECK}`,64 timeout: Duration.seconds(20),65 memorySize: 512,66 architecture: lambda.Architecture.ARM_64,67 tracing: lambda.Tracing.ACTIVE,68 entry: path.join(__dirname, "..", "src", "lambdas", "price-check.ts"),69 environment: {70 TABLE_NAME: dynamoDbTablePrices.tableName,71 },72 }73 );74 const lambdaNotification = new lambdaNodejs.NodejsFunction(75 this,76 RESOURCE_ID.LAMBDA_NOTOFICATION,77 {78 functionName: `${id}-${RESOURCE_ID.LAMBDA_NOTOFICATION}`,79 timeout: Duration.seconds(20),80 memorySize: 512,81 architecture: lambda.Architecture.ARM_64,82 tracing: lambda.Tracing.ACTIVE,83 entry: path.join(84 __dirname,85 "..",86 "src",87 "lambdas",88 "price-notification.ts"89 ),90 }91 );92 const lambdaPriceCheckAlarm = new cloudwatch.Alarm(93 this,94 RESOURCE_ID.LAMBDA_PRICE_CHECK_ALARM,95 {96 alarmName: `${id}-Errors-${RESOURCE_ID.LAMBDA_PRICE_CHECK_ALARM}`,97 metric: lambdaPriceCheck.metricErrors({98 period: Duration.hours(2),99 statistic: "max",100 }),101 threshold: 0,102 evaluationPeriods: 1,103 comparisonOperator:104 cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,105 treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,106 }107 );108 const lambdaNotificationAlarm = new cloudwatch.Alarm(109 this,110 RESOURCE_ID.LAMBDA_NOTOFICATION_ALARM,111 {112 alarmName: `${id}-Errors-${RESOURCE_ID.LAMBDA_NOTOFICATION_ALARM}`,113 metric: lambdaNotification.metricErrors({114 period: Duration.minutes(15),115 statistic: "max",116 }),117 threshold: 0,118 evaluationPeriods: 1,119 comparisonOperator:120 cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,121 treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,122 }123 );124 [lambdaPriceCheckAlarm, lambdaNotificationAlarm].forEach((alarm) => {125 alarm.addAlarmAction(new cloudwatchActions.SnsAction(snsTopicAlerts));126 alarm.addInsufficientDataAction(127 new cloudwatchActions.SnsAction(snsTopicAlerts)128 );129 alarm.addOkAction(new cloudwatchActions.SnsAction(snsTopicAlerts));130 });131 lambdaNotification.addEventSource(132 new awsLambdaEventSources.DynamoEventSource(dynamoDbTablePrices, {133 startingPosition: lambda.StartingPosition.TRIM_HORIZON,134 batchSize: 1,135 })136 );137 lambdaNotification.addToRolePolicy(138 new iam.PolicyStatement({139 actions: ["ses:SendEmail"],140 resources: ["*"],141 })142 );143 dynamoDbTablePrices.grantReadWriteData(lambdaPriceCheck);144 new events.Rule(this, RESOURCE_ID.SCHEDULED_EVENT, {145 ruleName: `${id}-${RESOURCE_ID.SCHEDULED_EVENT}`,146 schedule: events.Schedule.rate(Duration.minutes(15)),147 targets: [new targets.LambdaFunction(lambdaPriceCheck)],148 });149 }...

Using AI Code Generation

1var wpt = require('webpagetest');2var api = new wpt('www.webpagetest.org');3 if (err) return console.error(err);4 console.log(data.response.data.testId);5 api.getTestResults(data.response.data.testId, function(err, data) {6 if (err) return console.error(err);7 console.log(data.response.data.median.firstView);8 });9});10var wpt = require('webpagetest');11var api = new wpt('www.webpagetest.org');12 if (err) return console.error(err);13 console.log(data.response.data.testId);14 api.getTestResults(data.response.data.testId, function(err, data) {15 if (err) return console.error(err);16 console.log(data.response.data.median.firstView);17 });18});19var wpt = require('webpagetest');20var api = new wpt('www.webpagetest.org');21 if (err) return console.error(err);22 console.log(data.response.data.testId);23 api.getTestResults(data.response.data.testId, function(err, data) {24 if (err) return console.error(err);25 console.log(data.response.data.median.firstView);26 });27});28var wpt = require('webpagetest');29var api = new wpt('www.webpagetest.org');30 if (err) return console.error(err);31 console.log(data.response.data.testId);32 api.getTestResults(data.response.data.testId, function(err, data) {33 if (err) return console.error(err);34 console.log(data.response.data.median.firstView);35 });36});37var wpt = require('webpagetest');38var api = new wpt('www.webpagetest.org');39 if (

Using AI Code Generation

1var wpt = require('webpagetest');2var client = wpt('www.webpagetest.org');3var options = {4};5client.runTest(options, function(err, data) {6 if (err) {7 console.log(err);8 } else {9 console.log(data);10 client.getTestResults(data.data.testId, function(err, data) {11 if (err) {12 console.log(err);13 } else {14 console.log(data);15 }16 });17 }18});191. Fork it (

Using AI Code Generation

1var wpt = require('./lib/wpt');2var wpt = new WebPageTest('www.webpagetest.org');3wpt.getLocations(function(err, data) {4 if (err) {5 console.log(err);6 } else {7 console.log(data);8 }9});

Using AI Code Generation

1var wpt = require('webpagetest');2var webPageTest = new wpt('API_KEY');3var url = 'URL_TO_TEST';4var options = {5 videoParams: {6 }7};8webPageTest.runTest(url, options, function(err, data) {9 if (err) return console.error(err);10 var testId = data.data.testId;11 console.log('Test ID: ' + testId);12 webPageTest.getResourceId(testId, function(err, data) {13 if (err) return console.error(err);14 console.log(data);15 });16});

Using AI Code Generation

1var wpt = require('./wpt.js');2var wpt = new wpt('your_api_key');3wpt.getTestResults('test_id', 'resource_id', function(data) {4 console.log(data);5});6var wpt = require('./wpt.js');7var wpt = new wpt('your_api_key');8wpt.testStatus('test_id', function(data) {9 console.log(data);10});11var wpt = require('./wpt.js');12var wpt = new wpt('your_api_key');13wpt.testCancel('test_id', function(data) {14 console.log(data);15});16var wpt = require('./wpt.js');17var wpt = new wpt('your_api_key');18wpt.testResults('test_id', function(data) {19 console.log(data);20});21var wpt = require('./wpt.js');22var wpt = new wpt('your_api_key');23wpt.getLocations(function(data) {24 console.log(data);25});26var wpt = require('./wpt.js');27var wpt = new wpt('your_api_key');28wpt.getTesters(function(data) {29 console.log(data);30});31var wpt = require('./wpt.js');32var wpt = new wpt('your_api_key');33wpt.getLocations(function(data) {34 console.log(data);35});

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

• JUnit Tutorial
• TestNG Tutorial
• Webdriver Tutorial
• WebDriverIO Tutorial
• Protractor Tutorial
• Selenium 4 Tutorial
• Jenkins Tutorial
• NUnit Tutorial
• Jest Tutorial
• Playwright Tutorial
• Cypress Tutorial
• PyTest Tutorial

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run wpt automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

View sample repo View DocumentationSign up Free

›

_