Intended for automatic generation, web UI, and Automation NOT for manual pipeline building
Intended manual usage in IDE, all settings should have a class for either Dictionary or Pipeline format, so why not use the configuration class directly
No difference with previuos one, regarding functionality, but much more manageable when defining, and modifying
Pros
Cons
Pros
Cons
Pros
Cons
from utils.constants import DEFAULT_ENGINE_NAME PIPELINE = { 'enable': True, 'stages': [ { "type": "HighlightStage", "enable": True, "name": "highlight", "save_to_intermediate": False, "expand_result": False, "ui_only": None, 'engine_name': DEFAULT_ENGINE_NAME, "pre_tags": [ "<strong>" ], "pos_tags": [ "</strong>" ], "tags_schema": None, "force_source": False, "require_field_match": True, "number_of_fragments": 100, "fragment_size": 100, "order": None, "fields": [ { "field": "title", "fragment_size": None, "number_of_fragments": 1, "highlight_query": None, "type": None, "force_source": False }, { "field": "overview", "fragment_size": None, "number_of_fragments": 5, "highlight_query": None, "type": None, "force_source": False }, { "field": "url", "fragment_size": None, "number_of_fragments": 1, "highlight_query": None, "type": None, "force_source": False } ] }, { "aggs": [ { "type": "term", "field": "metadata.genres.name.keyword", "negated": False, "must": True, "id": "genres", "ui_only": { "display_name": "Genres", "hide_count": False, "hide_if_one": False, "translate": None, "max_display": 5, "mime_icon": False, "expanded": True, "enable_negative": True }, "aggs": [ { "id": "top_hits", "type": "top_hits" }, { "type": "term", "field": "metadata.spoken_languages.name.keyword", "negated": False, "must": True, "id": "spoken_languages", "ui_only": { "display_name": "Spoken Languages", "hide_count": False, "hide_if_one": False, "translate": None, "max_display": 5, "mime_icon": False, "expanded": False, "enable_negative": False }, "aggs": None, "max_values": None, "multi_select": False, "operation": "or", "order": None, "exclude_terms": None, "include_terms": None }], "max_values": 100, "multi_select": True, "operation": "or", "order": None, "exclude_terms": None, "include_terms": None }, { "type": "bool", "field": "adult", "negated": False, "must": True, "id": "adult", "ui_only": { "display_name": "Adult", "hide_count": False, "hide_if_one": False, "translate": None, "display_True": "True", "display_False": "False" }, "aggs": None }, { "type": "date_histogram", "field": "release_date", "negated": False, "must": True, "id": "release_date_h", "ui_only": { "display_name": "Release Date", "hide_if_one": False, "hide_count": False, "translate": None, "level": 0 }, "aggs": None, "calendar_interval": ["1y", "1q", "1M", "1d"], "format": ["yyyy", "'Q'q", "MMMM", "E dd"], "time_zone": ["-06:00"], "min_doc_count": [100, 1], "offset": None, "order": None }, { "type": "histogram", "field": "metadata.budget", "negated": False, "must": True, "id": "budget", "ui_only": { "display_name": "Budget", "hide_if_one": False, "hide_count": False, "translate": None, "level": 0 }, "aggs": None, "interval": [10000000, 1000000], "min_doc_count": [100, 10, 1], "order": None, "offset": None }, { "type": "slider", "field": "metadata.budget", "negated": False, "must": True, "id": "budget_slider", "ui_only": { "display_name": "Budget", "hide_count": False, "hide_if_one": False, "translate": None, "min": None, "max": None, "show_ticks": True }, "aggs": None }, { "type": "date_range", "field": "release_date", "negated": False, "must": True, "id": "release_date_dr", "ui_only": { "display_name": "Release Range", "hide_count": False, "hide_if_one": False, "translate": None }, "aggs": None, "format": "MM-yyyy", "ranges": [ { "key": "minus_10", "start": "now-10y", "start_display": None, "end": None, "end_display": None }, { "key": None, "start": "01-1960", "start_display": None, "end": "01-1970", "end_display": None }, { "key": None, "start": None, "start_display": None, "end": "now-10M/M", "end_display": None } ] }, { "type": "range", "field": "metadata.budget", "negated": False, "must": True, "id": "budget_r", "ui_only": { "display_name": "Budget Range", "hide_count": False, "hide_if_one": False, "translate": None }, "aggs": None, "ranges": [ { "key": "< 10,000,000", "start": None, "end": "10000000" }, { "key": "> 10,000,000", "start": "10000000", "end": None }, { "key": "50 to 100 millions", "start": "50000000", "end": "100000000" } ], "multi_select": False } ], "type": "DynamicAggStage", "enable": True, "name": "aggregations", "save_to_intermediate": False, "expand_result": False, "ui_only": None, "filters_name": "aggregations_filters", 'engine_name': DEFAULT_ENGINE_NAME, "search_response": "search", "ui_aggregations": True }, { "type": "FilterStage", "enable": True, "name": "filters", "engine_name": DEFAULT_ENGINE_NAME, "save_to_intermediate": False, "expand_result": False, "filters": [ { "type": "term", "field": "adult", "negated": False, "must": True, "value": False, "multi_select": False } ] }, { 'type': 'QueryStage', 'engine_name': DEFAULT_ENGINE_NAME, 'enable': True, 'qpl_enable': True, 'name': 'search', 'index': 'movies', 'fields': ['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], 'range_fields': ['metadata.release_date'], 'fetch_fields': ['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], 'implicit_operator': 'or', 'aggregations': 'aggregations', 'aggregations_filters': 'aggregations_filters', 'ui_only': { 'sort': { 'default': { 'field': '_score', 'display_name': 'Score', 'order': 'desc' }, 'options': [ { 'field': '_score', 'display_name': 'Score', 'order': 'desc' }, { 'field': 'release_date', 'display_name': 'Release Date', 'order': 'asc' }, { 'field': 'metadata.budget', 'display_name': 'Budget', 'order': 'desc' } ] }, 'page_size': { 'default': 25, 'options': [25, 50, 100] } }, 'sort': { 'field': '_score', 'order': 'desc' }, 'page_size_default': 25, 'highlight_stage': 'highlight' }, { 'type': 'DynamicResultsStage', 'engine_name': DEFAULT_ENGINE_NAME, 'enable': True, 'name': 'hits', 'title': { 'type': 'url', 'field': 'title', 'link': 'url' }, 'sub_title': { 'type': 'text', 'field': 'url', }, 'thumbnail': None, 'non_displayed_fields': [], 'header': [], 'body': [ [{'type': 'bool', 'label': 'Adult', 'field': 'adult'}, {'type': 'number', 'label': 'Popularity', 'field': 'metadata.popularity', 'format': '1.2'} ], [{'type': 'text', 'field': 'overview'}], [{'type': 'text', 'label': 'Status', 'field': 'status'}, {'type': 'date', 'label': 'Release Date', 'field': 'release_date', 'format': '%Y-%m-%d'}], [{'type': 'thumbnails', 'field': 'metadata.cast[*].profile_path_w185', 'alt': 'metadata.cast[*].name', 'detail': 'metadata.cast[*].name', 'full_size': 'metadata.cast[*].profile_path' }] ], 'metadata': { 'General': { 'fields': [ { 'type': 'chip', 'label': 'Chip', 'field': 'metadata.genres.name', 'unique': True }, {'type': 'text', 'field': 'title', 'label': 'Title'} ], 'sort': 'asc' }, }, 'search_response': 'search' }, ] }
from app.pipeline import Pipeline from app.pipeline.stages import HighlightStage, DynamicAggStage, FilterStage, \ DynamicResultsStage, QueryStage from app.pipeline.stages.query import QueryStageUIConfig from framework.aggregations import * from framework.fields import * from framework.filters import * from models.engines import HighlightField, SortConfig, PageSizeConfig, SortEntry from models.utils import SortOrder from utils.constants import DEFAULT_ENGINE_NAME SEARCH_STAGE_NAME = 'search' ######################################################################################################################## # PIPELINE STRUCTURE ######################################################################################################################## PIPELINE = Pipeline( enable=True, stages=[ HighlightStage( enable=True, name='highlight', engine_name=DEFAULT_ENGINE_NAME, pre_tags=['<strong>'], pos_tags=['</strong>'], tags_schema=None, force_source=False, require_field_match=True, number_of_fragments=3, fragment_size=100, order=None, fields=[ HighlightField(field='title', number_of_fragments=1), HighlightField(field='overview', number_of_fragments=5), HighlightField(field='url', number_of_fragments=1), ] ), FilterStage( enable=True, name='filters', engine_name=DEFAULT_ENGINE_NAME, save_to_intermediate=False, expand_result=False, filters=[ TermFilter(field='adult', negated=False, must=True, value=False, multi_select=False) ] ), DynamicAggStage( enable=True, name='aggregations', filters_name='aggregations_filters', save_to_intermediate=False, expand_result=False, engine_name=DEFAULT_ENGINE_NAME, search_response=SEARCH_STAGE_NAME, ui_aggregations=True, # This over here generates the aggregations for ui, if you don't want them, set it to False aggs=[ TermAgg( id='genres', field='metadata.genres.name.keyword', must=True, max_values=100, multi_select=True, operation='or', order=None, exclude_terms=None, include_terms=None, ui_only=TermAggUI(display_name='Genres', max_display=5, expanded=True), aggs=[ TopHitsAgg( id='top_hits', type='top_hits' ), TermAgg( field='metadata.spoken_languages.name.keyword', negated=False, must=True, id='spoken_languages', ui_only=TermAggUI(display_name='Spoken Languages', max_display=5, expanded=False), aggs=None, max_values=None, multi_select=False, operation='or', order=None, exclude_terms=None, include_terms=None )] ), BoolAgg( id='adult', field='adult', negated=False, must=True, ui_only=BoolAggUI(display_name='Adult', display_True='True', display_False='False') ), DateHistogramAgg( field='release_date', negated=False, must=True, id='release_date_h', ui_only=DateHistogramAggUI(display_name='Release Date'), calendar_interval=['1y', '1q', '1M', '1d'], format=['yyyy', '\'Q\'q', 'MMMM', 'E dd'], time_zone=['-06:00'], min_doc_count=[100, 1], offset=None, order=None ), HistogramAgg( id='budget', field='metadata.budget', negated=False, must=True, interval=[10000000, 1000000], min_doc_count=[100, 10, 1], order=None, offset=None, ui_only=HistogramAggUI(display_name='Budget') ), SliderAgg( id='budget_slider', field='metadata.budget', negated=False, must=True, ui_only=SliderAggUI(display_name='Budget', translate=None, min=None, max=None, show_ticks=True), ), DateRangeAgg( field='release_date', negated=False, must=True, id='release_date_dr', ui_only=DateRangeAggUI( display_name='Release Range', hide_count=False, translate=None ), format='MM-yyyy', ranges=[ DateRange( key='minus_10', start='now-10y', start_display=None, end=None, end_display=None ), DateRange( key=None, start='01-1960', start_display=None, end='01-1970', end_display=None ), DateRange( key=None, start=None, start_display=None, end='now-10M/M', end_display=None ) ] ), RangeAgg( field='metadata.budget', negated=False, must=True, id='budget_r', multi_select=False, ui_only=RangeAggUI( display_name='Budget Range', hide_count=False, translate=None ), ranges=[ Range( key='< 10,000,000', start=None, end='10000000' ), Range( key='> 10,000,000', start='10000000', end=None ), Range( key='50 to 100 millions', start='50000000', end='100000000' ) ], ) ] ), DynamicResultsStage( engine_name=DEFAULT_ENGINE_NAME, enable=True, name='hits', title=UrlField(field='title', link='url'), sub_title=TextField(field='url'), thumbnail=None, non_displayed_fields=[], body=[ [BoolField(label='Adult', field='adult'), NumberField(label='Popularity', field='metadata.popularity', format='1.2')], [TextField(field='overview')], [TextField(label='Status', field='status'), DateField(label='Release Date', field='release_date', format='%Y-%m-%d')], [ThumbnailsField(field='metadata.cast[*].profile_path_w185', alt='metadata.cast[*].name', detail='metadata.cast[*].name', full_size='metadata.cast[*].profile_path')] ], search_response=SEARCH_STAGE_NAME ), QueryStage( engine_name=DEFAULT_ENGINE_NAME, enable=True, qpl_enable=True, name=SEARCH_STAGE_NAME, index='movies', wildcard=False, fields=['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], range_fields=['metadata.budget'], date_fields=['release_date'], fetch_fields=['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], exclude_fields=None, implicit_operator='or', page_size_default=25, sort=SortEntry( field='_score', order=SortOrder.DESC ), aggregations='aggregations', aggregations_filters='aggregations_filters', highlight='highlight_stage', filters='filters', ui_only=QueryStageUIConfig( sort=SortConfig( default=SortEntry( field='_score', display_name='Score', order=SortOrder.DESC ), options=[ SortEntry( field='_score', display_name='Score', order=SortOrder.DESC ), SortEntry( field='release_date', display_name='Release Date', order=SortOrder.ASC ), SortEntry( field='metadata.budget', display_name='Budget', order=SortOrder.DESC ) ] ), page_size=PageSizeConfig( default=25, options=[25, 50, 100] ) ) ) ] )
from app.pipeline import Pipeline from app.pipeline.stages import HighlightStage, DynamicAggStage, FilterStage, \ DynamicResultsStage, QueryStage from app.pipeline.stages.query import QueryStageUIConfig from framework.aggregations import * from framework.fields import * from framework.filters import * from models.engines import HighlightField, SortConfig, PageSizeConfig, SortEntry from models.utils import SortOrder from utils.constants import DEFAULT_ENGINE_NAME ######################################################################################################################## # STAGES CONFIGURATION ######################################################################################################################## SEARCH_STAGE_NAME = 'search' ################# # HIGHLIGHT ################# _highlight_stage = HighlightStage( enable=True, name='highlight', engine_name=DEFAULT_ENGINE_NAME, pre_tags=['<strong>'], pos_tags=['</strong>'], tags_schema=None, force_source=False, require_field_match=True, number_of_fragments=3, fragment_size=100, order=None, fields=[ HighlightField(field='title', number_of_fragments=1), HighlightField(field='overview', number_of_fragments=5), HighlightField(field='url', number_of_fragments=1), ] ) ################# # AGGREGATIONS ################# _aggregation_stage = DynamicAggStage( enable=True, name='aggregations', filters_name='aggregations_filters', save_to_intermediate=False, expand_result=False, engine_name=DEFAULT_ENGINE_NAME, search_response=SEARCH_STAGE_NAME, ui_aggregations=True, # This over here generates the aggregations for ui, if you don't want them, set it to False aggs=[ TermAgg( id='genres', field='metadata.genres.name.keyword', must=True, max_values=100, multi_select=True, operation='or', order=None, exclude_terms=None, include_terms=None, ui_only=TermAggUI(display_name='Genres', max_display=5, expanded=True), aggs=[ TopHitsAgg( id='top_hits', type='top_hits' ), TermAgg( field='metadata.spoken_languages.name.keyword', negated=False, must=True, id='spoken_languages', ui_only=TermAggUI(display_name='Spoken Languages', max_display=5, expanded=False), aggs=None, max_values=None, multi_select=False, operation='or', order=None, exclude_terms=None, include_terms=None )] ), BoolAgg( id='adult', field='adult', negated=False, must=True, ui_only=BoolAggUI(display_name='Adult', display_True='True', display_False='False') ), DateHistogramAgg( field='release_date', negated=False, must=True, id='release_date_h', ui_only=DateHistogramAggUI(display_name='Release Date'), calendar_interval=['1y', '1q', '1M', '1d'], format=['yyyy', '\'Q\'q', 'MMMM', 'E dd'], time_zone=['-06:00'], min_doc_count=[100, 1], offset=None, order=None ), HistogramAgg( id='budget', field='metadata.budget', negated=False, must=True, interval=[10000000, 1000000], min_doc_count=[100, 10, 1], order=None, offset=None, ui_only=HistogramAggUI(display_name='Budget') ), SliderAgg( id='budget_slider', field='metadata.budget', negated=False, must=True, ui_only=SliderAggUI(display_name='Budget', translate=None, min=None, max=None, show_ticks=True), ), DateRangeAgg( field='release_date', negated=False, must=True, id='release_date_dr', ui_only=DateRangeAggUI( display_name='Release Range', hide_count=False, translate=None ), format='MM-yyyy', ranges=[ DateRange( key='minus_10', start='now-10y', start_display=None, end=None, end_display=None ), DateRange( key=None, start='01-1960', start_display=None, end='01-1970', end_display=None ), DateRange( key=None, start=None, start_display=None, end='now-10M/M', end_display=None ) ] ), RangeAgg( field='metadata.budget', negated=False, must=True, id='budget_r', multi_select=False, ui_only=RangeAggUI( display_name='Budget Range', hide_count=False, translate=None ), ranges=[ Range(key='< 10,000,000', start=None, end='10000000'), Range(key='> 10,000,000', start='10000000', end=None), Range(key='50 to 100 millions', start='50000000', end='100000000') ], ) ] ) ################# # FILTER ################# _filter_stage = FilterStage( enable=True, name='filters', engine_name=DEFAULT_ENGINE_NAME, save_to_intermediate=False, expand_result=False, filters=[ TermFilter(field='adult', negated=False, must=True, value=False, multi_select=False) ] ) ################# # UI RESULTS ################# _dynamic_results = DynamicResultsStage( engine_name=DEFAULT_ENGINE_NAME, enable=True, name='hits', title=UrlField(field='title', link='url'), sub_title=TextField(field='url'), thumbnail=None, non_displayed_fields=[], body=[ [BoolField(label='Adult', field='adult'), NumberField(label='Popularity', field='metadata.popularity', format='1.2')], [TextField(field='overview')], [TextField(label='Status', field='status'), DateField(label='Release Date', field='release_date', format='%Y-%m-%d')], [ThumbnailsField(field='metadata.cast[*].profile_path_w185', alt='metadata.cast[*].name', detail='metadata.cast[*].name', full_size='metadata.cast[*].profile_path')] ], search_response=SEARCH_STAGE_NAME ) ################# # SEARCH ################# _query_stage = QueryStage( engine_name=DEFAULT_ENGINE_NAME, enable=True, qpl_enable=True, name=SEARCH_STAGE_NAME, index='movies', wildcard=False, fields=['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], range_fields=['metadata.budget'], date_fields=['release_date'], fetch_fields=['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], exclude_fields=None, implicit_operator='or', page_size_default=25, sort=SortEntry( field='_score', order=SortOrder.DESC ), aggregations=_aggregation_stage.name, aggregations_filters=_aggregation_stage.filters_name, highlight=_highlight_stage.name, filters=_filter_stage.name, ui_only=QueryStageUIConfig( sort=SortConfig( default=SortEntry( field='_score', display_name='Score', order=SortOrder.DESC ), options=[ SortEntry( field='_score', display_name='Score', order=SortOrder.DESC ), SortEntry( field='release_date', display_name='Release Date', order=SortOrder.ASC ), SortEntry( field='metadata.budget', display_name='Budget', order=SortOrder.DESC ) ] ), page_size=PageSizeConfig( default=25, options=[25, 50, 100] ) ) ) ######################################################################################################################## # PIPELINE STRUCTURE ######################################################################################################################## PIPELINE = Pipeline( enable=True, stages=[ _highlight_stage, _filter_stage, _aggregation_stage, _dynamic_results, _query_stage ] )