You are viewing an old version of this page. View the current version.
Compare with Current View Page History
« Previous Version 7 Next »
All Pipeline:
Intended for automatic generation, web UI, and Automation NOT for manual pipeline building
Don't use this format
Intended manual usage in IDE, all settings should have a class for either Dictionary or Pipeline format, so why not use the configuration class directly
No difference with previuos one, regarding functionality, but much more manageable when defining, and modifying
Pros
Cons
Pros
Cons
Pros
Cons
from utils.constants import DEFAULT_ENGINE_NAME PIPELINE = { 'enable': True, 'stages': [ { "type": "HighlightStage", "enable": True, "name": "highlight", "save_to_intermediate": False, "expand_result": False, "ui_only": None, 'engine_name': DEFAULT_ENGINE_NAME, "pre_tags": [ "<strong>" ], "pos_tags": [ "</strong>" ], "tags_schema": None, "force_source": False, "require_field_match": True, "number_of_fragments": 100, "fragment_size": 100, "order": None, "fields": [ { "field": "title", "fragment_size": None, "number_of_fragments": 1, "highlight_query": None, "type": None, "force_source": False }, { "field": "overview", "fragment_size": None, "number_of_fragments": 5, "highlight_query": None, "type": None, "force_source": False }, { "field": "url", "fragment_size": None, "number_of_fragments": 1, "highlight_query": None, "type": None, "force_source": False } ] }, { "aggs": [ { "type": "term", "field": "metadata.genres.name.keyword", "negated": False, "must": True, "id": "genres", "ui_only": { "display_name": "Genres", "hide_count": False, "hide_if_one": False, "translate": None, "max_display": 5, "mime_icon": False, "expanded": True, "enable_negative": True }, "aggs": [ { "id": "top_hits", "type": "top_hits" }, { "type": "term", "field": "metadata.spoken_languages.name.keyword", "negated": False, "must": True, "id": "spoken_languages", "ui_only": { "display_name": "Spoken Languages", "hide_count": False, "hide_if_one": False, "translate": None, "max_display": 5, "mime_icon": False, "expanded": False, "enable_negative": False }, "aggs": None, "max_values": None, "multi_select": False, "operation": "or", "order": None, "exclude_terms": None, "include_terms": None }], "max_values": 100, "multi_select": True, "operation": "or", "order": None, "exclude_terms": None, "include_terms": None }, { "type": "bool", "field": "adult", "negated": False, "must": True, "id": "adult", "ui_only": { "display_name": "Adult", "hide_count": False, "hide_if_one": False, "translate": None, "display_True": "True", "display_False": "False" }, "aggs": None }, { "type": "date_histogram", "field": "release_date", "negated": False, "must": True, "id": "release_date_h", "ui_only": { "display_name": "Release Date", "hide_if_one": False, "hide_count": False, "translate": None, "level": 0 }, "aggs": None, "calendar_interval": ["1y", "1q", "1M", "1d"], "format": ["yyyy", "'Q'q", "MMMM", "E dd"], "time_zone": ["-06:00"], "min_doc_count": [100, 1], "offset": None, "order": None }, { "type": "histogram", "field": "metadata.budget", "negated": False, "must": True, "id": "budget", "ui_only": { "display_name": "Budget", "hide_if_one": False, "hide_count": False, "translate": None, "level": 0 }, "aggs": None, "interval": [10000000, 1000000], "min_doc_count": [100, 10, 1], "order": None, "offset": None }, { "type": "slider", "field": "metadata.budget", "negated": False, "must": True, "id": "budget_slider", "ui_only": { "display_name": "Budget", "hide_count": False, "hide_if_one": False, "translate": None, "min": None, "max": None, "show_ticks": True }, "aggs": None }, { "type": "date_range", "field": "release_date", "negated": False, "must": True, "id": "release_date_dr", "ui_only": { "display_name": "Release Range", "hide_count": False, "hide_if_one": False, "translate": None }, "aggs": None, "format": "MM-yyyy", "ranges": [ { "key": "minus_10", "start": "now-10y", "start_display": None, "end": None, "end_display": None }, { "key": None, "start": "01-1960", "start_display": None, "end": "01-1970", "end_display": None }, { "key": None, "start": None, "start_display": None, "end": "now-10M/M", "end_display": None } ] }, { "type": "range", "field": "metadata.budget", "negated": False, "must": True, "id": "budget_r", "ui_only": { "display_name": "Budget Range", "hide_count": False, "hide_if_one": False, "translate": None }, "aggs": None, "ranges": [ { "key": "< 10,000,000", "start": None, "end": "10000000" }, { "key": "> 10,000,000", "start": "10000000", "end": None }, { "key": "50 to 100 millions", "start": "50000000", "end": "100000000" } ], "multi_select": False } ], "type": "DynamicAggStage", "enable": True, "name": "aggregations", "save_to_intermediate": False, "expand_result": False, "ui_only": None, "filters_name": "aggregations_filters", 'engine_name': DEFAULT_ENGINE_NAME, "search_response": "search", "ui_aggregations": True }, { "type": "FilterStage", "enable": True, "name": "filters", "engine_name": DEFAULT_ENGINE_NAME, "save_to_intermediate": False, "expand_result": False, "filters": [ { "type": "term", "field": "adult", "negated": False, "must": True, "value": False, "multi_select": False } ] }, { 'type': 'QueryStage', 'engine_name': DEFAULT_ENGINE_NAME, 'enable': True, 'qpl_enable': True, 'name': 'search', 'index': 'movies', 'fields': ['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], 'range_fields': ['metadata.release_date'], 'fetch_fields': ['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], 'implicit_operator': 'or', 'aggregations': 'aggregations', 'aggregations_filters': 'aggregations_filters', 'ui_only': { 'sort': { 'default': { 'field': '_score', 'display_name': 'Score', 'order': 'desc' }, 'options': [ { 'field': '_score', 'display_name': 'Score', 'order': 'desc' }, { 'field': 'release_date', 'display_name': 'Release Date', 'order': 'asc' }, { 'field': 'metadata.budget', 'display_name': 'Budget', 'order': 'desc' } ] }, 'page_size': { 'default': 25, 'options': [25, 50, 100] } }, 'sort': { 'field': '_score', 'order': 'desc' }, 'page_size_default': 25, 'highlight_stage': 'highlight' }, { 'type': 'DynamicResultsStage', 'engine_name': DEFAULT_ENGINE_NAME, 'enable': True, 'name': 'hits', 'title': { 'type': 'url', 'field': 'title', 'link': 'url' }, 'sub_title': { 'type': 'text', 'field': 'url', }, 'thumbnail': None, 'non_displayed_fields': [], 'header': [], 'body': [ [{'type': 'bool', 'label': 'Adult', 'field': 'adult'}, {'type': 'number', 'label': 'Popularity', 'field': 'metadata.popularity', 'format': '1.2'} ], [{'type': 'text', 'field': 'overview'}], [{'type': 'text', 'label': 'Status', 'field': 'status'}, {'type': 'date', 'label': 'Release Date', 'field': 'release_date', 'format': '%Y-%m-%d'}], [{'type': 'thumbnails', 'field': 'metadata.cast[*].profile_path_w185', 'alt': 'metadata.cast[*].name', 'detail': 'metadata.cast[*].name', 'full_size': 'metadata.cast[*].profile_path' }] ], 'metadata': { 'General': { 'fields': [ { 'type': 'chip', 'label': 'Chip', 'field': 'metadata.genres.name', 'unique': True }, {'type': 'text', 'field': 'title', 'label': 'Title'} ], 'sort': 'asc' }, }, 'search_response': 'search' }, ] }
from app.pipeline import Pipeline from app.pipeline.stages import HighlightStage, DynamicAggStage, FilterStage, \ DynamicResultsStage, QueryStage from app.pipeline.stages.query import QueryStageUIConfig from framework.aggregations import * from framework.fields import * from framework.filters import * from models.engines import HighlightField, SortConfig, PageSizeConfig, SortEntry from models.utils import SortOrder from utils.constants import DEFAULT_ENGINE_NAME SEARCH_STAGE_NAME = 'search' ######################################################################################################################## # PIPELINE STRUCTURE ######################################################################################################################## PIPELINE = Pipeline( enable=True, stages=[ HighlightStage( enable=True, name='highlight', engine_name=DEFAULT_ENGINE_NAME, pre_tags=['<strong>'], pos_tags=['</strong>'], tags_schema=None, force_source=False, require_field_match=True, number_of_fragments=3, fragment_size=100, order=None, fields=[ HighlightField(field='title', number_of_fragments=1), HighlightField(field='overview', number_of_fragments=5), HighlightField(field='url', number_of_fragments=1), ] ), FilterStage( enable=True, name='filters', engine_name=DEFAULT_ENGINE_NAME, save_to_intermediate=False, expand_result=False, filters=[ TermFilter(field='adult', negated=False, must=True, value=False, multi_select=False) ] ), DynamicAggStage( enable=True, name='aggregations', filters_name='aggregations_filters', save_to_intermediate=False, expand_result=False, engine_name=DEFAULT_ENGINE_NAME, search_response=SEARCH_STAGE_NAME, ui_aggregations=True, # This over here generates the aggregations for ui, if you don't want them, set it to False aggs=[ TermAgg( id='genres', field='metadata.genres.name.keyword', must=True, max_values=100, multi_select=True, operation='or', order=None, exclude_terms=None, include_terms=None, ui_only=TermAggUI(display_name='Genres', max_display=5, expanded=True), aggs=[ TopHitsAgg( id='top_hits', type='top_hits' ), TermAgg( field='metadata.spoken_languages.name.keyword', negated=False, must=True, id='spoken_languages', ui_only=TermAggUI(display_name='Spoken Languages', max_display=5, expanded=False), aggs=None, max_values=None, multi_select=False, operation='or', order=None, exclude_terms=None, include_terms=None )] ), BoolAgg( id='adult', field='adult', negated=False, must=True, ui_only=BoolAggUI(display_name='Adult', display_True='True', display_False='False') ), DateHistogramAgg( field='release_date', negated=False, must=True, id='release_date_h', ui_only=DateHistogramAggUI(display_name='Release Date'), calendar_interval=['1y', '1q', '1M', '1d'], format=['yyyy', '\'Q\'q', 'MMMM', 'E dd'], time_zone=['-06:00'], min_doc_count=[100, 1], offset=None, order=None ), HistogramAgg( id='budget', field='metadata.budget', negated=False, must=True, interval=[10000000, 1000000], min_doc_count=[100, 10, 1], order=None, offset=None, ui_only=HistogramAggUI(display_name='Budget') ), SliderAgg( id='budget_slider', field='metadata.budget', negated=False, must=True, ui_only=SliderAggUI(display_name='Budget', translate=None, min=None, max=None, show_ticks=True), ), DateRangeAgg( field='release_date', negated=False, must=True, id='release_date_dr', ui_only=DateRangeAggUI( display_name='Release Range', hide_count=False, translate=None ), format='MM-yyyy', ranges=[ DateRange( key='minus_10', start='now-10y', start_display=None, end=None, end_display=None ), DateRange( key=None, start='01-1960', start_display=None, end='01-1970', end_display=None ), DateRange( key=None, start=None, start_display=None, end='now-10M/M', end_display=None ) ] ), RangeAgg( field='metadata.budget', negated=False, must=True, id='budget_r', multi_select=False, ui_only=RangeAggUI( display_name='Budget Range', hide_count=False, translate=None ), ranges=[ Range( key='< 10,000,000', start=None, end='10000000' ), Range( key='> 10,000,000', start='10000000', end=None ), Range( key='50 to 100 millions', start='50000000', end='100000000' ) ], ) ] ), DynamicResultsStage( engine_name=DEFAULT_ENGINE_NAME, enable=True, name='hits', title=UrlField(field='title', link='url'), sub_title=TextField(field='url'), thumbnail=None, non_displayed_fields=[], body=[ [BoolField(label='Adult', field='adult'), NumberField(label='Popularity', field='metadata.popularity', format='1.2')], [TextField(field='overview')], [TextField(label='Status', field='status'), DateField(label='Release Date', field='release_date', format='%Y-%m-%d')], [ThumbnailsField(field='metadata.cast[*].profile_path_w185', alt='metadata.cast[*].name', detail='metadata.cast[*].name', full_size='metadata.cast[*].profile_path')] ], search_response=SEARCH_STAGE_NAME ), QueryStage( engine_name=DEFAULT_ENGINE_NAME, enable=True, qpl_enable=True, name=SEARCH_STAGE_NAME, index='movies', wildcard=False, fields=['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], range_fields=['metadata.budget'], date_fields=['release_date'], fetch_fields=['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], exclude_fields=None, implicit_operator='or', page_size_default=25, sort=SortEntry( field='_score', order=SortOrder.DESC ), aggregations='aggregations', aggregations_filters='aggregations_filters', highlight='highlight_stage', filters='filters', ui_only=QueryStageUIConfig( sort=SortConfig( default=SortEntry( field='_score', display_name='Score', order=SortOrder.DESC ), options=[ SortEntry( field='_score', display_name='Score', order=SortOrder.DESC ), SortEntry( field='release_date', display_name='Release Date', order=SortOrder.ASC ), SortEntry( field='metadata.budget', display_name='Budget', order=SortOrder.DESC ) ] ), page_size=PageSizeConfig( default=25, options=[25, 50, 100] ) ) ) ] )
from app.pipeline import Pipeline from app.pipeline.stages import HighlightStage, DynamicAggStage, FilterStage, \ DynamicResultsStage, QueryStage from app.pipeline.stages.query import QueryStageUIConfig from framework.aggregations import * from framework.fields import * from framework.filters import * from models.engines import HighlightField, SortConfig, PageSizeConfig, SortEntry from models.utils import SortOrder from utils.constants import DEFAULT_ENGINE_NAME ######################################################################################################################## # STAGES CONFIGURATION ######################################################################################################################## SEARCH_STAGE_NAME = 'search' ################# # HIGHLIGHT ################# _highlight_stage = HighlightStage( enable=True, name='highlight', engine_name=DEFAULT_ENGINE_NAME, pre_tags=['<strong>'], pos_tags=['</strong>'], tags_schema=None, force_source=False, require_field_match=True, number_of_fragments=3, fragment_size=100, order=None, fields=[ HighlightField(field='title', number_of_fragments=1), HighlightField(field='overview', number_of_fragments=5), HighlightField(field='url', number_of_fragments=1), ] ) ################# # AGGREGATIONS ################# _aggregation_stage = DynamicAggStage( enable=True, name='aggregations', filters_name='aggregations_filters', save_to_intermediate=False, expand_result=False, engine_name=DEFAULT_ENGINE_NAME, search_response=SEARCH_STAGE_NAME, ui_aggregations=True, # This over here generates the aggregations for ui, if you don't want them, set it to False aggs=[ TermAgg( id='genres', field='metadata.genres.name.keyword', must=True, max_values=100, multi_select=True, operation='or', order=None, exclude_terms=None, include_terms=None, ui_only=TermAggUI(display_name='Genres', max_display=5, expanded=True), aggs=[ TopHitsAgg( id='top_hits', type='top_hits' ), TermAgg( field='metadata.spoken_languages.name.keyword', negated=False, must=True, id='spoken_languages', ui_only=TermAggUI(display_name='Spoken Languages', max_display=5, expanded=False), aggs=None, max_values=None, multi_select=False, operation='or', order=None, exclude_terms=None, include_terms=None )] ), BoolAgg( id='adult', field='adult', negated=False, must=True, ui_only=BoolAggUI(display_name='Adult', display_True='True', display_False='False') ), DateHistogramAgg( field='release_date', negated=False, must=True, id='release_date_h', ui_only=DateHistogramAggUI(display_name='Release Date'), calendar_interval=['1y', '1q', '1M', '1d'], format=['yyyy', '\'Q\'q', 'MMMM', 'E dd'], time_zone=['-06:00'], min_doc_count=[100, 1], offset=None, order=None ), HistogramAgg( id='budget', field='metadata.budget', negated=False, must=True, interval=[10000000, 1000000], min_doc_count=[100, 10, 1], order=None, offset=None, ui_only=HistogramAggUI(display_name='Budget') ), SliderAgg( id='budget_slider', field='metadata.budget', negated=False, must=True, ui_only=SliderAggUI(display_name='Budget', translate=None, min=None, max=None, show_ticks=True), ), DateRangeAgg( field='release_date', negated=False, must=True, id='release_date_dr', ui_only=DateRangeAggUI( display_name='Release Range', hide_count=False, translate=None ), format='MM-yyyy', ranges=[ DateRange( key='minus_10', start='now-10y', start_display=None, end=None, end_display=None ), DateRange( key=None, start='01-1960', start_display=None, end='01-1970', end_display=None ), DateRange( key=None, start=None, start_display=None, end='now-10M/M', end_display=None ) ] ), RangeAgg( field='metadata.budget', negated=False, must=True, id='budget_r', multi_select=False, ui_only=RangeAggUI( display_name='Budget Range', hide_count=False, translate=None ), ranges=[ Range(key='< 10,000,000', start=None, end='10000000'), Range(key='> 10,000,000', start='10000000', end=None), Range(key='50 to 100 millions', start='50000000', end='100000000') ], ) ] ) ################# # FILTER ################# _filter_stage = FilterStage( enable=True, name='filters', engine_name=DEFAULT_ENGINE_NAME, save_to_intermediate=False, expand_result=False, filters=[ TermFilter(field='adult', negated=False, must=True, value=False, multi_select=False) ] ) ################# # UI RESULTS ################# _dynamic_results = DynamicResultsStage( engine_name=DEFAULT_ENGINE_NAME, enable=True, name='hits', title=UrlField(field='title', link='url'), sub_title=TextField(field='url'), thumbnail=None, non_displayed_fields=[], body=[ [BoolField(label='Adult', field='adult'), NumberField(label='Popularity', field='metadata.popularity', format='1.2')], [TextField(field='overview')], [TextField(label='Status', field='status'), DateField(label='Release Date', field='release_date', format='%Y-%m-%d')], [ThumbnailsField(field='metadata.cast[*].profile_path_w185', alt='metadata.cast[*].name', detail='metadata.cast[*].name', full_size='metadata.cast[*].profile_path')] ], search_response=SEARCH_STAGE_NAME ) ################# # SEARCH ################# _query_stage = QueryStage( engine_name=DEFAULT_ENGINE_NAME, enable=True, qpl_enable=True, name=SEARCH_STAGE_NAME, index='movies', wildcard=False, fields=['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], range_fields=['metadata.budget'], date_fields=['release_date'], fetch_fields=['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], exclude_fields=None, implicit_operator='or', page_size_default=25, sort=SortEntry( field='_score', order=SortOrder.DESC ), aggregations=_aggregation_stage.name, aggregations_filters=_aggregation_stage.filters_name, highlight=_highlight_stage.name, filters=_filter_stage.name, ui_only=QueryStageUIConfig( sort=SortConfig( default=SortEntry( field='_score', display_name='Score', order=SortOrder.DESC ), options=[ SortEntry( field='_score', display_name='Score', order=SortOrder.DESC ), SortEntry( field='release_date', display_name='Release Date', order=SortOrder.ASC ), SortEntry( field='metadata.budget', display_name='Budget', order=SortOrder.DESC ) ] ), page_size=PageSizeConfig( default=25, options=[25, 50, 100] ) ) ) ######################################################################################################################## # PIPELINE STRUCTURE ######################################################################################################################## PIPELINE = Pipeline( enable=True, stages=[ _highlight_stage, _filter_stage, _aggregation_stage, _dynamic_results, _query_stage ] )
1) Create the pipeline file
Start by creating a python file for the pipeline configuration in the folder config/pipeline
A descriptive name works the best
On The File
from app.pipeline import Pipeline from app.pipeline.stages import DemoStage
_demo = DemoStage( enable=True, name='demo', save_to_intermediate=False, expand_result=False, ui_only=None, halt_on_exception=False )
PIPELINE = Pipeline( enable=True, stages=[ _demo ] )
You can go to /es/docs and execute the GET pipeline endpoint and see if the name appears