from datetime import datetime, timedelta from unittest import TestCase from mock import patch, sentinel, PropertyMock, MagicMock from elasticsearch import NotFoundError as ElasticsearchNotFoundError from werkzeug.exceptions import NotFound, BadRequest from distribution import app from distribution.models.feed import FeedItem from distribution.models.article import Article from distribution.models.taxonomy import TaxonomyTopic from . import ElasticsearchMock, get_document from .. import utils GREEN_HEALTH = { 'cluster_name': 'elasticsearch', 'status': 'green', 'timed_out': False, 'number_of_nodes': 1, 'number_of_data_nodes': 1, 'active_primary_shards': 5, 'active_shards': 5, 'relocating_shards': 0, 'initializing_shards': 0, 'unassigned_shards': 0, 'number_of_pending_tasks': 0 } YELLOW_HEALTH = { 'cluster_name': 'elasticsearch', 'status': 'yellow', 'timed_out': False, 'number_of_nodes': 1, 'number_of_data_nodes': 1, 'active_primary_shards': 5, 'active_shards': 5, 'relocating_shards': 0, 'initializing_shards': 0, 'unassigned_shards': 4, 'number_of_pending_tasks': 0 } GROUP_BY_SOURCES = {'group_by_sources': { 'buckets': [ {'doc_count': 333, 'key': 'dummy-3'}, {'doc_count': 22, 'key': 'dummy-2'}, {'doc_count': 1, 'key': 'dummy-1'}], 'doc_count_error_upper_bound': 0, 'sum_other_doc_count': 0}} RFC3339_FORMAT = '%Y-%m-%dT%H:%M:%SZ' def taxonomy_topic_mget_mock(keys): objects = [] for key in keys: objects.append(TaxonomyTopic({ 'uuid': key, 'name': 'Fake topic', 'topic_type': 'locations', 'creation_datetime': str(datetime.now()), 'updated_datetime': str(datetime.now()) })) return objects def get_item_base_document(**kwargs): source = utils.get_stored_item(**kwargs) doc = { '_index': 'distribution_feed', '_type': 'feeditem', '_source': source } doc['_id'] = doc['_source']['id'] return doc def get_item_document(**kwargs): return get_document(get_item_base_document(**kwargs)) def get_item_documents(size): docs = [] for _ in xrange(size): docs.append(get_item_base_document(regenerate_id=True)) return docs def get_item_search_results(size): results = { 'took': 69, 'timed_out': False, '_shards': { 'total': 5, 'successful': 5, 'failed': 0 }, 'hits': { 'total': size, 'max_score': 1 if size > 0 else None, 'hits': [] } } for _ in xrange(size): doc = get_item_base_document(regenerate_id=True) doc['score'] = 1 results['hits']['hits'].append(doc) return results def get_aggregated_search_result(): return dict(get_item_search_results(0), aggregations=GROUP_BY_SOURCES) def get_pointed_indices(indices): result = {} for i in indices: result[i] = {'aliases': {'distribution_feed': {}}} return result class ElasticsearchTestCase(TestCase): def setUp(self): self.manager = FeedItem.manager self.manager.backend._client = ElasticsearchMock() @patch('elasticsearch.Elasticsearch.get', return_value=get_item_document()) @patch('distribution.models.feed.TaxonomyTopic.manager.get', return_value=None) def test_get_existent(self, taxonomy_get_mock, get_mock): item = self.manager.get('123456') self.assertEqual(item.data['title'], 'Desperately Seeking Queen Nefertiti') get_mock.assert_called_once_with('distribution_feed', '123456') @patch('elasticsearch.Elasticsearch.get', side_effect=ElasticsearchNotFoundError) def test_get_missing(self, get_mock): with self.assertRaises(NotFound): self.manager.get('123456') get_mock.assert_called_once_with('distribution_feed', '123456') @patch('elasticsearch.Elasticsearch.search', return_value=get_item_search_results(1)) def test_get_by_field_existent(self, get_mock): uri = '150302-honduras-lost-city-monkey-god-maya-ancient-archaeology' item = self.manager.get_by_field('uri', uri) self.assertEqual(item.data['title'], 'Desperately Seeking Queen Nefertiti') query_mock = { "query": { "bool": { "must": { "match": { "uri": { "query": "{}".format(uri), "type": "phrase" } } }, "should": [ { "match": { "meta.is_deleted": "false" } } ] } } } get_mock.assert_called_once_with('distribution_feed', '', query_mock, size=1) @patch('elasticsearch.Elasticsearch.search', return_value=get_item_search_results(0)) def test_get_by_field_missing(self, get_mock): with self.assertRaises(NotFound): uri = '150302-honduras-lost-city-monkey-god-maya-ancient-archaeology' self.manager.get_by_field('uri', uri) query_mock = { "query": { "bool": { "must": { "match": { "uri": { "query": "{}".format(uri), "type": "phrase" } } }, "should": [ { "match": { "meta.is_deleted": "false" } } ] } } } get_mock.assert_called_once_with('distribution_feed', '', query_mock, size=1) @patch('elasticsearch.Elasticsearch.search', return_value=get_item_search_results(5)) @patch('distribution.models.feed.TaxonomyTopic.manager.get', return_value=None) def test_get_all(self, get_mock, search_mock): items = self.manager.search({}) self.assertEqual(search_mock.call_count, 1) self.assertEqual(len(items), 5) for i in items: self.assertIsInstance(i, FeedItem) @patch('distribution.models.feed.FeedItem.manager.aggregated_search', return_value=GROUP_BY_SOURCES) def test_count_sources(self, aggregated_search_mock): result = self.manager.count_sources(sentinel.QUERY) aggregated_search_mock.assert_called_once_with(sentinel.QUERY, group_by="sources") self.assertEquals(result, {'dummy-3': 333, 'dummy-2': 22, 'dummy-1': 1}) @patch('distribution.models.feed.FeedItem.manager._search', return_value=(sentinel.QUERY2, get_aggregated_search_result())) def test_aggregated_search(self, _search_mock): with self.assertRaises(TypeError): self.manager.aggregated_search(sentinel.QUERY1) result = self.manager.aggregated_search(sentinel.QUERY1, group_by=sentinel.GROUPBY) _search_mock.assert_called_once_with(sentinel.QUERY1, group_by=sentinel.GROUPBY) self.assertEquals(result, GROUP_BY_SOURCES) @patch('distribution.models.feed.FeedItem.manager.build_search_query') def test__search(self, build_search_query_mock): build_search_query_mock.return_value = sentinel.BUILT_QUERY doc_type_owner = utils.get_attr_owner(self.manager, 'doc_type') doc_type_prop_mock = PropertyMock(return_value=sentinel.DOC_TYPE) doc_type_patcher = patch.object(doc_type_owner, 'doc_type', new_callable=doc_type_prop_mock) backend_owner = utils.get_attr_owner(self.manager, 'backend') backend_mock = MagicMock(**{ 'client.search.return_value': sentinel.ES_RESULT}) backend_prop_mock = PropertyMock(return_value=backend_mock) backend_patcher = patch.object(backend_owner, 'backend', new_callable=backend_prop_mock) with doc_type_patcher: with backend_patcher: result = self.manager._search(sentinel.QUERY, group_by=sentinel.GROUPBY) build_search_query_mock.assert_called_once_with( sentinel.QUERY, group_by=sentinel.GROUPBY) backend_mock.client.search.assert_called_once_with( FeedItem.manager.index, '', sentinel.BUILT_QUERY) self.assertEquals(result, (sentinel.BUILT_QUERY, sentinel.ES_RESULT)) def test_cast_to_model(self): feed_class_mock = MagicMock(FeedItem) article_class_mock = MagicMock(Article) hits_mock = [{'_source': sentinel.SOURCE_1, '_type': 'feeditem'}, {'_source': sentinel.SOURCE_2, '_type': 'article'}] with patch('distribution.managers.get_model', side_effect=[feed_class_mock, article_class_mock]): result = FeedItem.manager.cast_to_models(hits_mock) self.assertEqual(len(result), 2) feed_class_mock.assert_called_once_with(sentinel.SOURCE_1, validate=False) article_class_mock.assert_called_once_with(sentinel.SOURCE_2, validate=False) @patch('distribution.models.feed.FeedItem.manager._search', return_value=(sentinel.BUILT_QUERY, {'hits': {'hits': [sentinel.HITS]}})) @patch('distribution.models.feed.FeedItem.manager.cast_to_models', return_value=sentinel.RESULT) def test_search(self, cast_to_model_mock, _search_mock): result = self.manager.search(sentinel.QUERY, option=sentinel.OPTION) self.assertEqual(sentinel.RESULT, result) cast_to_model_mock.assert_called_once_with([sentinel.HITS]) _search_mock.assert_called_once_with(sentinel.QUERY, option=sentinel.OPTION) @patch('distribution.managers.feed.scan', return_value=get_item_documents(0)) @patch('distribution.managers.backends.elastic.bulk') def test_purge_empty(self, bulk_mock, scan_mock): self.manager.purge('news', '*', '*') self.assertEqual(scan_mock.call_count, 1) self.assertEqual(len(bulk_mock.call_args[0][1]), 0) @patch('distribution.managers.feed.scan', return_value=get_item_documents(30)) @patch('distribution.managers.backends.elastic.bulk') def test_purge_nonempty(self, bulk_mock, scan_mock): self.manager.purge('news', '*', '*') self.assertEqual(scan_mock.call_count, 1) self.assertEqual(len(bulk_mock.call_args[0][1]), 30) action = bulk_mock.call_args[0][1][0] self.assertEqual(action['_op_type'], 'delete') self.assertEqual(action['_index'], 'distribution_feed') self.assertEqual(action['_type'], 'feeditem') self.assertIn('_id', action) @patch('distribution.managers.backends.elastic.bulk', return_value=([], [])) def test_disable_many(self, bulk_mock): self.manager.disable_many(['fba75b9c-3dcb-4ea7-bd32-9b3716d65692', 'fba75b9c-3dcb-4ea7-bd32-9b3716d65693', 'fba75b9c-3dcb-4ea7-bd32-9b3716d65694']) self.assertEqual(len(bulk_mock.call_args[0][1]), 3) self.assertEqual(bulk_mock.call_args[0][1][0], { '_op_type': 'update', '_index': 'distribution_feed', '_type': 'feeditem', '_id': 'fba75b9c-3dcb-4ea7-bd32-9b3716d65692', 'doc': {'meta': {'is_deleted': True}} }) @patch('distribution.managers.backends.elastic.bulk') def test_delete(self, bulk_mock): self.manager.delete(['fba75b9c-3dcb-4ea7-bd32-9b3716d65692', 'fba75b9c-3dcb-4ea7-bd32-9b3716d65693', 'fba75b9c-3dcb-4ea7-bd32-9b3716d65694']) self.assertEqual(len(bulk_mock.call_args[0][1]), 3) self.assertEqual(bulk_mock.call_args[0][1][0], { '_op_type': 'delete', '_index': 'distribution_feed', '_type': 'feeditem', '_id': 'fba75b9c-3dcb-4ea7-bd32-9b3716d65692' }) @patch('elasticsearch.Elasticsearch.index') @patch('distribution.models.feed.TaxonomyTopic.manager.get', return_value=None) @patch('distribution.models.feed.FeedItem.manager.search_by_field', side_effect=NotFound) def test_save(self, search_by_field_mock, get_mock, index_mock): item = FeedItem(utils.get_stored_item()) self.manager.save(item) index_mock.assert_called_once_with('distribution_feed', 'feeditem', item.data, item.key) @patch('distribution.managers.backends.elastic.bulk') @patch('distribution.models.feed.TaxonomyTopic.manager.get', return_value=None) @patch('distribution.models.feed.FeedItem.manager.search_by_field', side_effect=NotFound) def test_bulk_save(self, search_by_field_mock, get_mock, bulk_mock): item = FeedItem(utils.get_stored_item()) self.manager.bulk_save([item], []) self.assertEqual(len(bulk_mock.call_args[0][1]), 1) self.assertEqual(bulk_mock.call_args[0][1][0]['_op_type'], 'index') self.assertEqual(bulk_mock.call_args[0][1][0]['_id'], item.key) @patch('elasticsearch.client.cluster.ClusterClient.health', return_value=GREEN_HEALTH) @patch('elasticsearch.client.indices.IndicesClient.get_alias', return_value=get_pointed_indices(['distribution_feed_aaaaaaaaaa'])) def test_health_all_green(self, get_alias_mock, health_mock): self.manager.backend.indices = { 'distribution_feed': { 'real_index_name': 'distribution_feed_aaaaaaaaaa' } } health = self.manager.backend.health() self.assertEqual(health['status'], 'green') self.assertEqual(health['alias_health']['distribution_feed']['status'], 'green') self.assertEqual(health['current_index_health']['distribution_feed']['status'], 'green') self.assertEqual(health['cluster_health']['status'], 'green') self.assertEqual(health['current_index_name'], ['distribution_feed_aaaaaaaaaa']) self.assertEqual(health['alias']['distribution_feed']['status'], 'green') self.assertEqual( health['alias']['distribution_feed']['pointed_indices'], ['distribution_feed_aaaaaaaaaa']) @patch('elasticsearch.client.cluster.ClusterClient.health', return_value=GREEN_HEALTH) @patch('elasticsearch.client.indices.IndicesClient.get_alias', return_value=get_pointed_indices(['distribution_feed_bbbbbbbbbb'])) def test_health_wrong_alias(self, get_alias_mock, health_mock): self.manager.backend.indices = { 'distribution_feed': { 'real_index_name': 'distribution_feed_aaaaaaaaaa' } } health = self.manager.backend.health() self.assertEqual(health['status'], 'red') self.assertEqual(health['alias_health']['distribution_feed']['status'], 'green') self.assertEqual(health['current_index_health']['distribution_feed']['status'], 'green') self.assertEqual(health['cluster_health']['status'], 'green') self.assertEqual(health['current_index_name'], ['distribution_feed_aaaaaaaaaa']) self.assertEqual(health['alias']['distribution_feed']['status'], 'red') self.assertEqual( health['alias']['distribution_feed']['pointed_indices'], ['distribution_feed_bbbbbbbbbb']) @patch('elasticsearch.client.cluster.ClusterClient.health', return_value=YELLOW_HEALTH) @patch('elasticsearch.client.indices.IndicesClient.get_alias', return_value=get_pointed_indices(['distribution_feed_aaaaaaaaaa'])) def test_health_cluster_yellow(self, get_alias_mock, health_mock): self.manager.backend.indices = { 'distribution_feed': { 'real_index_name': 'distribution_feed_aaaaaaaaaa' } } health = self.manager.backend.health() self.assertEqual(health['status'], 'yellow') self.assertEqual(health['alias_health']['distribution_feed']['status'], 'yellow') self.assertEqual(health['current_index_health']['distribution_feed']['status'], 'yellow') self.assertEqual(health['cluster_health']['status'], 'yellow') self.assertEqual(health['current_index_name'], ['distribution_feed_aaaaaaaaaa']) self.assertEqual(health['alias']['distribution_feed']['status'], 'green') self.assertEqual( health['alias']['distribution_feed']['pointed_indices'], ['distribution_feed_aaaaaaaaaa']) @patch('elasticsearch.client.cluster.ClusterClient.health', side_effect=Exception) @patch('elasticsearch.client.indices.IndicesClient.get_alias', return_value=get_pointed_indices(['distribution_feed_aaaaaaaaaa'])) def test_health_cluster_exception(self, get_alias_mock, health_mock): self.manager.backend.indices = { 'distribution_feed': { 'real_index_name': 'distribution_feed_aaaaaaaaaa' } } health = self.manager.backend.health() self.assertEqual(health['status'], 'red') self.assertEqual(health['alias_health']['distribution_feed']['status'], 'red') self.assertEqual(health['current_index_health']['distribution_feed']['status'], 'red') self.assertEqual(health['cluster_health']['status'], 'red') self.assertEqual(health['current_index_name'], ['distribution_feed_aaaaaaaaaa']) self.assertEqual(health['alias']['distribution_feed']['status'], 'green') self.assertEqual( health['alias']['distribution_feed']['pointed_indices'], ['distribution_feed_aaaaaaaaaa']) class ElasticsearchCreateQueryTestCase(TestCase): def setUp(self): self.manager = FeedItem.manager def get_initial_query(self): return { 'query': { 'bool': { 'must': { 'bool': { 'must': [], 'must_not': [] } }, "filter": { "bool": { "must": [], "must_not": [] } } } }, 'sort': [{ 'publication_datetime': { 'order': 'desc' } }] } def test_build_datetime_query_with_publication_datetime_from(self): arguments = { 'publication_datetime__from': '2015-08-13T18:30:02Z', 'publication_datetime': None } query = self.get_initial_query() self.manager._build_query_datetime(query, self.manager.parse_query(arguments, {}), 'publication_datetime') self.assertEqual(query['query']['bool']['filter']['bool'], { 'must': [ { 'range': { 'publication_datetime': { 'gte': '2015-08-13T18:30:02Z', 'lte': 'now' } } } ], 'must_not': [] }) def test_build_datetime_query_with_publication_datetime_from_to(self): arguments = { 'publication_datetime__from': '2015-08-13T18:30:02Z', 'publication_datetime__to': '2015-08-20T18:30:02Z', 'publication_datetime': None } query = self.get_initial_query() self.manager._build_query_datetime(query, self.manager.parse_query(arguments, {}), 'publication_datetime') self.assertEqual(query['query']['bool']['filter']['bool'], { 'must': [ { 'range': { 'publication_datetime': { 'gte': '2015-08-13T18:30:02Z', 'lte': '2015-08-20T18:30:02Z' } } } ], 'must_not': [] }) def test_build_datetime_query_by_day(self): arguments = { 'publication_datetime__from': None, 'publication_datetime__to': None, 'publication_datetime': '7d' } query = self.get_initial_query() self.manager._build_query_datetime(query, self.manager.parse_query(arguments, {}), 'publication_datetime') difference = (datetime.utcnow() - timedelta(days=int(arguments.get('publication_datetime')[:-1]))) difference_str = difference.strftime(RFC3339_FORMAT) self.assertEqual(query['query']['bool']['filter']['bool'], { 'must': [ { 'range': { 'publication_datetime': { 'gte': difference_str, 'lte': 'now' } } } ], 'must_not': [] }) def test_build_datetime_query_by_hour_publication_datetime(self): arguments = { 'publication_datetime__from': None, 'publication_datetime__to': None, 'publication_datetime': '5h' } query = self.get_initial_query() self.manager._build_query_datetime(query, self.manager.parse_query(arguments, {}), 'publication_datetime') difference = (datetime.utcnow() - timedelta(hours=int(arguments.get('publication_datetime')[:-1]))) difference_str = difference.strftime(RFC3339_FORMAT) self.assertEqual(query['query']['bool']['filter']['bool'], { 'must': [ { 'range': { 'publication_datetime': { 'gte': difference_str, 'lte': 'now' } } } ], 'must_not': [] }) def test_build_datetime_query_by_hour_last_modified_datetime(self): arguments = { 'last_modified_datetime__from': None, 'last_modified_datetime__to': None, 'last_modified_datetime': '5h' } query = self.get_initial_query() self.manager._build_query_datetime(query, self.manager.parse_query(arguments, {}), 'last_modified_datetime') difference = (datetime.utcnow() - timedelta(hours=int(arguments.get('last_modified_datetime')[:-1]))) difference_str = difference.strftime(RFC3339_FORMAT) self.assertEqual(query['query']['bool']['filter']['bool'], { 'must': [ { 'range': { 'last_modified_datetime': { 'gte': difference_str, 'lte': 'now' } } } ], 'must_not': [] }) def test_build_datetime_without_arguments_publication_datetime(self): arguments = { 'publication_datetime__from': None, 'publication_datetime__to': None, 'publication_datetime': None } query = self.get_initial_query() self.manager._build_query_datetime(query, self.manager.parse_query(arguments, {}), 'publication_datetime') difference = datetime.utcnow() - timedelta(days=self.manager.MAX_DAYS) difference_str = difference.strftime(RFC3339_FORMAT) self.assertEqual(query['query']['bool']['filter']['bool'], { 'must': [ { 'range': { 'publication_datetime': { 'gte': difference_str, 'lte': 'now' } } } ], 'must_not': [] }) def test_build_datetime_without_arguments_last_modified_datetime(self): arguments = { 'publication_datetime__from': None, 'publication_datetime__to': None, 'publication_datetime': None } query = self.get_initial_query() self.manager._build_query_datetime(query, self.manager.parse_query(arguments, {}), 'last_modified_datetime') difference = datetime.utcnow() - timedelta(days=self.manager.MAX_DAYS) difference_str = difference.strftime(RFC3339_FORMAT) self.assertEqual(query['query']['bool']['filter']['bool'], { 'must': [ { 'range': { 'last_modified_datetime': { 'gte': difference_str, 'lte': 'now' } } } ], 'must_not': [] }) def test_build_datetime_with_future_content_false(self): current_date = datetime.today() future_date = (current_date + timedelta(days=30)) arguments = { 'publication_datetime__from': current_date.strftime(RFC3339_FORMAT), 'publication_datetime__to': future_date.strftime(RFC3339_FORMAT) } self.assertRaises(BadRequest, self.manager.parse_query, arguments, {}) def test_build_datetime_with_future_content_true_in_arguments(self): current_date = datetime.today() future_date = (current_date + timedelta(days=30)) arguments = { 'publication_datetime__from': current_date.strftime(RFC3339_FORMAT), 'publication_datetime__to': future_date.strftime(RFC3339_FORMAT), 'future_content': True } self.assertRaises(BadRequest, self.manager.parse_query, arguments, {}) def test_build_datetime_with_future_content_true(self): current_date = datetime.today() future_date = (current_date + timedelta(days=30)) future_content = True arguments = { 'publication_datetime__from': current_date.strftime(RFC3339_FORMAT), 'publication_datetime__to': future_date.strftime(RFC3339_FORMAT) } permissions = { 'future_content': future_content } query = self.get_initial_query() self.manager._build_query_datetime(query, self.manager.parse_query(arguments, permissions, future_content), 'publication_datetime') self.assertEqual(query['query']['bool']['filter']['bool'], { 'must': [ { 'range': { 'publication_datetime': { 'gte': current_date.strftime(RFC3339_FORMAT), 'lte': future_date.strftime(RFC3339_FORMAT) } } } ], 'must_not': [] }) def get_all_should_conditions(self, parent_condition): should_conditions = [] for condition in parent_condition: if 'bool' in condition: for subcondition in condition['bool']['should']: if 'bool' in subcondition: should_conditions.extend(subcondition['bool']['should']) else: should_conditions.append(subcondition) return should_conditions @patch('distribution.models.feed.TaxonomyTopic.manager.mget', new=taxonomy_topic_mget_mock) def test_build_parameters_query_with_filters(self): arguments = { 'locations': ('4a54af42-def9-3f39-8b67-9480dab24c94 AND ' '407852b7-85a0-341a-84a8-0e5b3c3dd517'), 'sources__not': 'bb2f1ff5-29a9-35ed-ae94-a2daec5610e8', 'subjects': ('770660e2-a945-3148-b579-3af86e517768 OR ' '338a0f50-ac7f-35ec-8c69-1e335efa438e'), 'persons__not': ('a60a3daf-bd0f-3bae-ad90-cc6b91b37a8b OR ' 'c5c0ade3-8997-337c-87d4-f172f9a60357'), 'content_type': 'photo,gallery' } query = self.get_initial_query() self.manager._build_query_content(query, self.manager.parse_query(arguments, {})) self.assertIn({ 'bool': { 'should': [{ 'match_phrase': { 'sources': 'bb2f1ff5-29a9-35ed-ae94-a2daec5610e8' } }], 'minimum_should_match': 1 } }, query['query']['bool']['must']['bool']['must_not']) self.assertIn({ 'bool': { 'should': [{ 'match_phrase': { 'locations': '4a54af42-def9-3f39-8b67-9480dab24c94' } }], 'minimum_should_match': 1 } }, query['query']['bool']['must']['bool']['must']) self.assertIn({ 'bool': { 'should': [{ 'match_phrase': { 'locations': '407852b7-85a0-341a-84a8-0e5b3c3dd517' } }], 'minimum_should_match': 1 } }, query['query']['bool']['must']['bool']['must']) must_should_conditions = self.get_all_should_conditions( query['query']['bool']['must']['bool']['must']) must_not_should_conditions = self.get_all_should_conditions( query['query']['bool']['must']['bool']['must_not']) self.assertEqual(len(must_should_conditions), 6) self.assertEqual(len(must_not_should_conditions), 3) self.assertIn({ 'match_phrase': { 'subjects': '770660e2-a945-3148-b579-3af86e517768' } }, must_should_conditions) self.assertIn({ 'match_phrase': { 'subjects': '338a0f50-ac7f-35ec-8c69-1e335efa438e' } }, must_should_conditions) self.assertIn({ 'match_phrase': { 'content_type': 'photo' } }, must_should_conditions) self.assertIn({ 'match_phrase': { 'content_type': 'gallery' } }, must_should_conditions) self.assertIn({ 'match_phrase': { 'persons': 'a60a3daf-bd0f-3bae-ad90-cc6b91b37a8b' } }, must_not_should_conditions) self.assertIn({ 'match_phrase': { 'persons': 'c5c0ade3-8997-337c-87d4-f172f9a60357' } }, must_not_should_conditions) @patch('distribution.models.feed.TaxonomyTopic.manager.mget', new=taxonomy_topic_mget_mock) def test_build_parameters_query_with_filters_and_with_permissions(self): arguments = { 'locations': ('4a54af42-def9-3f39-8b67-9480dab24c94 AND ' '407852b7-85a0-341a-84a8-0e5b3c3dd517'), 'sources': 'bb2f1ff5-29a9-35ed-ae94-a2daec5610e8', 'subjects': ('770660e2-a945-3148-b579-3af86e517768 OR ' '338a0f50-ac7f-35ec-8c69-1e335efa438e'), 'persons__not': ('a60a3daf-bd0f-3bae-ad90-cc6b91b37a8b OR ' 'c5c0ade3-8997-337c-87d4-f172f9a60357'), 'content_type': 'photo,gallery' } permissions = { 'sources': ('4a54af42-def9-3f39-8b67-9480dab24c94 OR ' 'bb2f1ff5-29a9-35ed-ae94-a2daec5610e8'), 'content_type': 'photo' } query = self.get_initial_query() self.manager._build_query_content(query, self.manager.parse_query(arguments, permissions)) self.assertIn({ 'bool': { 'minimum_should_match': 1, 'should': [{ 'match_phrase': { 'sources': 'bb2f1ff5-29a9-35ed-ae94-a2daec5610e8' } }] } }, query['query']['bool']['must']['bool']['must']) self.assertIn({ 'bool': { 'should': [{ 'match_phrase': { 'locations': '4a54af42-def9-3f39-8b67-9480dab24c94' } }], 'minimum_should_match': 1 } }, query['query']['bool']['must']['bool']['must']) self.assertIn({ 'bool': { 'should': [{ 'match_phrase': { 'locations': '407852b7-85a0-341a-84a8-0e5b3c3dd517' } }], 'minimum_should_match': 1 } }, query['query']['bool']['must']['bool']['must']) self.assertIn({ 'match_phrase': { 'content_type': 'photo' } }, query['query']['bool']['must']['bool']['must']) must_should_conditions = self.get_all_should_conditions( query['query']['bool']['must']['bool']['must']) must_not_should_conditions = self.get_all_should_conditions( query['query']['bool']['must']['bool']['must_not']) self.assertEqual(len(must_should_conditions), 5) self.assertEqual(len(must_not_should_conditions), 2) self.assertIn({ 'match_phrase': { 'subjects': '770660e2-a945-3148-b579-3af86e517768' } }, must_should_conditions) self.assertIn({ 'match_phrase': { 'subjects': '338a0f50-ac7f-35ec-8c69-1e335efa438e' } }, must_should_conditions) self.assertIn({ 'match_phrase': { 'persons': 'a60a3daf-bd0f-3bae-ad90-cc6b91b37a8b' } }, must_not_should_conditions) self.assertIn({ 'match_phrase': { 'persons': 'c5c0ade3-8997-337c-87d4-f172f9a60357' } }, must_not_should_conditions) @patch('distribution.models.feed.TaxonomyTopic.manager.mget', new=taxonomy_topic_mget_mock) def test_build_parameters_query_with_series_id_filter(self): arguments = { 'series.id': '111023', 'content_type': 'series,schedule' } query = self.get_initial_query() self.manager._build_query_content(query, self.manager.parse_query(arguments, {})) self.assertIn({ 'multi_match': { 'query': '111023', 'type': 'phrase', 'fields': ['series_id', 'episode.series_id'] } }, query['query']['bool']['must']['bool']['must']) def test_build_blacklist_source(self): original_source_blacklist = app.config['SOURCE_BLACKLIST'] app.config['SOURCE_BLACKLIST'] = ['2ea1233d-1a91-36a5-b06b-52819a1fd03c'] query = self.get_initial_query() self.manager._build_query_blacklist_source(query, self.manager.parse_query({}, {})) app.config['SOURCE_BLACKLIST'] = original_source_blacklist self.assertEqual(query['query']['bool']['must']['bool'], { 'must': [], 'must_not': [ { 'match_phrase': { 'sources': '2ea1233d-1a91-36a5-b06b-52819a1fd03c' } } ] }) def test_build_query_aggregation(self): query = dict(self.get_initial_query(), size=100) query_copy = query.copy() self.manager._build_query_aggregation(query) self.assertEqual(query, query_copy) self.manager._build_query_aggregation(query, group_by="foo") self.assertEqual(query['query'], query_copy['query']) self.assertEqual(query['sort'], query_copy['sort']) self.assertEqual(query['size'], 0) self.assertEqual(query['aggs'], {'group_by_foo': {"terms": {"field": 'foo'}}}) def test_add_pagination(self): query = self.get_initial_query() arguments = { 'limit': '5', 'page': '2' } self.manager._add_pagination(query, arguments) self.assertEqual(query['size'], 5) self.assertEqual(query['from'], 5) @staticmethod def get_from_date(days): difference = datetime.utcnow() - timedelta(days=days) return difference.strftime("%Y-%m-%dT%H:%M:%SZ") def test_build_query_with_not_default_datetime_field(self): arguments = {'uri': 'test_uri'} permissions = {'future_content': True} # When future content is True and we are not filtering by a datetime, method # should not add any kind of datetime to query query = self.manager.build_search_query(arguments=arguments, permissions=permissions) self.assertEqual(len(query['query']['bool']['filter']['bool']['must']), 1) self.assertNotIn('range', query['query']['bool']['filter']['bool']['must'][0]) # When future content is False and we are not filtering by a datetime, method # should add a publication datetime at the query query = self.manager.build_search_query(arguments=arguments) self.assertEqual(query['query']['bool']['filter']['bool']['must'][1], {'range': {'publication_datetime': {'lte': 'now'}}}) # When future content is True and we are filtering by a datetime different from publication # datetime, we will filter based on that datetime and won't add publication_datetime filter arguments['last_modified_datetime'] = '3d' query = self.manager.build_search_query(arguments=arguments, permissions=permissions) from_datetime = self.get_from_date(3) self.assertEqual(len(query['query']['bool']['filter']['bool']['must']), 2) self.assertEqual(query['query']['bool']['filter']['bool']['must'][1], {'range': {'last_modified_datetime': {'lte': 'now', 'gte': from_datetime}}}) # When future content is False and we are filtering by a datetime different from publication # datetime, we will add a publication_datetime_filter query = self.manager.build_search_query(arguments=arguments) from_datetime = self.get_from_date(3) self.assertEqual(len(query['query']['bool']['filter']['bool']['must']), 3) self.assertEqual(query['query']['bool']['filter']['bool']['must'][1], {'range': {'last_modified_datetime': {'lte': 'now', 'gte': from_datetime}}}) self.assertEqual(query['query']['bool']['filter']['bool']['must'][2], {'range': {'publication_datetime': {'lte': 'now'}}}) # When future content is False and we are filtering by publication_datetime, we only add # it one time and with the full range del arguments['last_modified_datetime'] arguments['publication_datetime'] = '3d' query = self.manager.build_search_query(arguments=arguments) from_datetime = self.get_from_date(3) self.assertEqual(len(query['query']['bool']['filter']['bool']['must']), 2) self.assertEqual(query['query']['bool']['filter']['bool']['must'][1], {'range': {'publication_datetime': {'lte': 'now', 'gte': from_datetime}}}) def test_build_query_with_language_permission(self): allowed_language = 'en-US' arguments = {'uri': 'test_uri'} permissions = {'language': allowed_language} query = self.manager.build_search_query(arguments=arguments, permissions=permissions) query_must = query['query']['bool']['must']['bool']['must'] self.assertEqual(len(query_must), 3) self.assertEqual(len(query_must[2]['bool']['should']), 1) self.assertIn('minimum_should_match', query_must[2]['bool']) self.assertEqual(query_must[2]['bool']['minimum_should_match'], 1) self.assertIn('minimum_should_match', query_must[2]['bool']['should'][0]['bool']) self.assertEqual( query_must[2]['bool']['should'][0]['bool']['should'][0]['match_phrase']['language'], allowed_language ) def test_build_query_with_language_permission_and_query_language(self): allowed_language = 'en-US' arguments = {'uri': 'test_uri', 'language': 'es'} permissions = {'language': allowed_language} query = self.manager.build_search_query(arguments=arguments, permissions=permissions) query_must = query['query']['bool']['must']['bool']['must'] self.assertEqual(len(query_must), 3) self.assertEqual(len(query_must[2]['bool']['should']), 1) self.assertIn('minimum_should_match', query_must[2]['bool']) self.assertEqual(query_must[2]['bool']['minimum_should_match'], 1) self.assertIn('minimum_should_match', query_must[2]['bool']['should'][0]['bool']) self.assertEqual( query_must[2]['bool']['should'][0]['bool']['should'][0]['match_phrase']['language'], allowed_language ) def test_build_query_with_multiple_language_permission(self): allowed_languages = 'en-US OR pt' arguments = {'uri': 'test_uri'} permissions = {'language': allowed_languages} query = self.manager.build_search_query(arguments=arguments, permissions=permissions) query_must = query['query']['bool']['must']['bool']['must'] self.assertEqual(len(query_must), 3) self.assertEqual(len(query_must[2]['bool']['should']), 2) self.assertIn('minimum_should_match', query_must[2]['bool']) self.assertEqual(query_must[2]['bool']['minimum_should_match'], 1) self.assertIn('minimum_should_match', query_must[2]['bool']['should'][0]['bool']) self.assertIn('minimum_should_match', query_must[2]['bool']['should'][1]['bool']) languages = allowed_languages.split(' OR ') self.assertEqual( query_must[2]['bool']['should'][0]['bool']['should'][0]['match_phrase']['language'], languages[0] ) self.assertEqual( query_must[2]['bool']['should'][1]['bool']['should'][0]['match_phrase']['language'], languages[1] ) def test_build_query_with_multiple_language_permission_and_query_language(self): allowed_languages = 'en-US OR es' arguments = {'uri': 'test_uri', 'language': 'es'} permissions = {'language': allowed_languages} query = self.manager.build_search_query(arguments=arguments, permissions=permissions) query_must = query['query']['bool']['must']['bool']['must'] self.assertEqual(len(query_must), 3) self.assertEqual(len(query_must[2]['bool']['should']), 1) self.assertIn('minimum_should_match', query_must[2]['bool']) self.assertEqual(query_must[2]['bool']['minimum_should_match'], 1) self.assertIn('minimum_should_match', query_must[2]['bool']['should'][0]['bool']) self.assertEqual( query_must[2]['bool']['should'][0]['bool']['should'][0]['match_phrase']['language'], allowed_languages.split(' OR ')[1] ) def test_build_query_with_default_datetime_field(self): arguments = {'content_type': 'article:story'} permissions = {'future_content': True} # When future_content is True and we don't filter by a datetime, we will add the default # filter query = self.manager.build_search_query(arguments=arguments, permissions=permissions) from_datetime = self.get_from_date(self.manager.MAX_DAYS) self.assertEqual(query['query']['bool']['filter']['bool']['must'][1], {'range': {'publication_datetime': {'lte': 'now', 'gte': from_datetime}}}) # When future_content is False and we don't filter by a datetime, we will add the default # filter query = self.manager.build_search_query(arguments=arguments) self.assertEqual(query['query']['bool']['filter']['bool']['must'][1], {'range': {'publication_datetime': {'lte': 'now', 'gte': from_datetime}}}) # When future_content is False and we filter by a datetime different to publication_datetime # we add a publication datetime filter with lte arguments['last_modified_datetime'] = '3d' from_datetime = self.get_from_date(3) query = self.manager.build_search_query(arguments=arguments) self.assertEqual(len(query['query']['bool']['filter']['bool']['must']), 3) self.assertEqual(query['query']['bool']['filter']['bool']['must'][1], {'range': {'last_modified_datetime': {'lte': 'now', 'gte': from_datetime}}}) self.assertEqual(query['query']['bool']['filter']['bool']['must'][2], {'range': {'publication_datetime': {'lte': 'now'}}}) # When future content is True and we filter by a datetime different to publication_datetime # we don't add a publication datetime filter arguments['last_modified_datetime'] = '3d' from_datetime = self.get_from_date(3) query = self.manager.build_search_query(arguments=arguments, permissions=permissions) self.assertEqual(len(query['query']['bool']['filter']['bool']['must']), 2) self.assertEqual(query['query']['bool']['filter']['bool']['must'][1], {'range': {'last_modified_datetime': {'lte': 'now', 'gte': from_datetime}}}) # When future content is False and we filter by publication_datetime, we only add one # datetime filter with full range del arguments['last_modified_datetime'] arguments['publication_datetime'] = '3d' query = self.manager.build_search_query(arguments=arguments) from_datetime = self.get_from_date(3) self.assertEqual(len(query['query']['bool']['filter']['bool']['must']), 2) self.assertEqual(query['query']['bool']['filter']['bool']['must'][1], {'range': {'publication_datetime': {'lte': 'now', 'gte': from_datetime}}}) def test_build_search_query(self): self.check_build_search_query(False, 'publication_datetime', 1) self.check_build_search_query('publication_datetime', 'publication_datetime', 1) self.check_build_search_query('last_modified_datetime', 'last_modified_datetime', 2) self.check_build_search_query('meta.last_update', 'meta.last_update', 2) @patch('distribution.models.feed.FeedItem.manager.parse_query', return_value=sentinel.PARSED_ARGUMENTS) @patch('distribution.models.feed.FeedItem.manager._add_pagination') @patch('distribution.models.feed.FeedItem.manager._build_query_content') @patch('distribution.models.feed.FeedItem.manager._build_query_datetime') @patch('distribution.models.feed.FeedItem.manager._build_query_blacklist_source') @patch('distribution.models.feed.FeedItem.manager._build_query_aggregation') def check_build_search_query(self, argument_field, expected_field, calls_to_bqdm, _build_query_aggregation_mock, _build_query_blacklist_source_mock, _build_query_datetime_mock, _build_query_content_mock, _add_pagination_mock, parse_query_mock): argument_field = {argument_field: True} \ if argument_field else {} with patch.object(self.manager, 'MAX_ROWS', sentinel.MAX_ROWS): result = self.manager.build_search_query(sentinel.ARG, arguments=argument_field, order=sentinel.ORDER, extra=sentinel.EXTRA) expected = { 'query': { 'bool': { 'must': { 'bool': { 'must': [], 'must_not': [] } }, "filter": { "bool": { "must": [ { 'term': { 'meta.is_deleted': False } }, ], "must_not": [ { 'exists': { 'field': 'meta.is_metadata' } } ] } } } }, 'sort': [{ expected_field: { 'order': sentinel.ORDER, 'unmapped_type': 'date', } }] } self.assertEqual(expected, result) _add_pagination_mock.assert_called_once_with( expected, argument_field) parse_query_mock.assert_called_once_with(argument_field, {}, False) _build_query_content_mock.assert_called_once_with( expected, sentinel.PARSED_ARGUMENTS) if expected_field != 'publication_datetime': _build_query_datetime_mock.assert_called_with( expected, sentinel.PARSED_ARGUMENTS, 'publication_datetime', from_range=None) else: _build_query_datetime_mock.assert_called_with( expected, sentinel.PARSED_ARGUMENTS, expected_field) self.assertEqual(_build_query_datetime_mock.call_count, calls_to_bqdm) _build_query_blacklist_source_mock.assert_called_once_with( expected, sentinel.PARSED_ARGUMENTS) _build_query_aggregation_mock.assert_called_once_with( expected, **{'arguments': argument_field, 'order': sentinel.ORDER, 'extra': sentinel.EXTRA}) def test_build_required_fields(self): arguments = { 'exists_fields': 'images' } query = self.get_initial_query() self.manager._build_exists_fields(query, arguments) self.assertEqual(query['query']['bool']['filter']['bool'], { 'must': [ { 'exists': { 'field': 'images' } } ], 'must_not': [] }) def build_metadata_fields(self, value): arguments = { 'metadata': value } query = self.get_initial_query() self.manager._build_query_metadata(query, arguments) return query def test_add_metadata_fields(self): query = self.build_metadata_fields('true') self.assertEqual(query['query']['bool']['filter']['bool'], { 'must': [ { 'exists': { 'field': 'meta.is_metadata' } } ], 'must_not': [] }) query = self.build_metadata_fields('false') self.assertEqual(query['query']['bool']['filter']['bool'], { 'must': [], 'must_not': [ { 'exists': { 'field': 'meta.is_metadata' } } ] }) self.assertRaises(ValueError, query=self.build_metadata_fields('foobar')) def test_build_query_title_abstract(self): arguments = { 'title-abstract': 'FooBar' } adition = { "dis_max": { "queries": [{ "match": { "title": { "query": 'FooBar', "operator": "and" } } }, { "match": { "abstract": { "query": 'FooBar', "operator": "and" } } }], "tie_breaker": 0.3 } } query = self.get_initial_query() self.manager._build_query_title_abstract(query, arguments) self.assertIn(adition, query['query']['bool']['must']['bool']['must'])