Intended for automatic generation, web UI, and Automation NOT for manual pipeline building
Warning |
---|
Don't use this format |
Intended manual usage in IDE, all settings should have a class for either Dictionary or Pipeline format, so why not use the configuration class directly
No difference with previuos one, regarding functionality, but much more manageable when defining, and modifying
Panel |
---|
Pros
Cons
|
Panel |
---|
Pros
Cons
|
Panel |
---|
Pros
Cons
|
Code Block | ||||||||||
---|---|---|---|---|---|---|---|---|---|---|
| ||||||||||
from utils.constants import DEFAULT_ENGINE_NAME PIPELINE = { 'enable': True, 'stages': [ { "type": "HighlightStage", "enable": True, "name": "highlight", "save_to_intermediate": False, "expand_result": False, "ui_only": None, 'engine_name': DEFAULT_ENGINE_NAME, "pre_tags": [ "<strong>" ], "pos_tags": [ "</strong>" ], "tags_schema": None, "force_source": False, "require_field_match": True, "number_of_fragments": 100, "fragment_size": 100, "order": None, "fields": [ { "field": "title", "fragment_size": None, "number_of_fragments": 1, "highlight_query": None, "type": None, "force_source": False }, { "field": "overview", "fragment_size": None, "number_of_fragments": 5, "highlight_query": None, "type": None, "force_source": False }, { "field": "url", "fragment_size": None, "number_of_fragments": 1, "highlight_query": None, "type": None, "force_source": False } ] }, { "aggs": [ { "type": "term", "field": "metadata.genres.name.keyword", "negated": False, "must": True, "id": "genres", "ui_only": { "display_name": "Genres", "hide_count": False, "hide_if_one": False, "translate": None, "max_display": 5, "mime_icon": False, "expanded": True, "enable_negative": True }, "aggs": [ { "id": "top_hits", "type": "top_hits" }, { "type": "term", "field": "metadata.spoken_languages.name.keyword", "negated": False, "must": True, "id": "spoken_languages", "ui_only": { "display_name": "Spoken Languages", "hide_count": False, "hide_if_one": False, "translate": None, "max_display": 5, "mime_icon": False, "expanded": False, "enable_negative": False }, "aggs": None, "max_values": None, "multi_select": False, "operation": "or", "order": None, "exclude_terms": None, "include_terms": None }], "max_values": 100, "multi_select": True, "operation": "or", "order": None, "exclude_terms": None, "include_terms": None }, { "type": "bool", "field": "adult", "negated": False, "must": True, "id": "adult", "ui_only": { "display_name": "Adult", "hide_count": False, "hide_if_one": False, "translate": None, "display_True": "True", "display_False": "False" }, "aggs": None }, { "type": "date_histogram", "field": "release_date", "negated": False, "must": True, "id": "release_date_h", "ui_only": { "display_name": "Release Date", "hide_if_one": False, "hide_count": False, "translate": None, "level": 0 }, "aggs": None, "calendar_interval": ["1y", "1q", "1M", "1d"], "format": ["yyyy", "'Q'q", "MMMM", "E dd"], "time_zone": ["-06:00"], "min_doc_count": [100, 1], "offset": None, "order": None }, { "type": "histogram", "field": "metadata.budget", "negated": False, "must": True, "id": "budget", "ui_only": { "display_name": "Budget", "hide_if_one": False, "hide_count": False, "translate": None, "level": 0 }, "aggs": None, "interval": [10000000, 1000000], "min_doc_count": [100, 10, 1], "order": None, "offset": None }, { "type": "slider", "field": "metadata.budget", "negated": False, "must": True, "id": "budget_slider", "ui_only": { "display_name": "Budget", "hide_count": False, "hide_if_one": False, "translate": None, "min": None, "max": None, "show_ticks": True }, "aggs": None }, { "type": "date_range", "field": "release_date", "negated": False, "must": True, "id": "release_date_dr", "ui_only": { "display_name": "Release Range", "hide_count": False, "hide_if_one": False, "translate": None }, "aggs": None, "format": "MM-yyyy", "ranges": [ { "key": "minus_10", "start": "now-10y", "start_display": None, "end": None, "end_display": None }, { "key": None, "start": "01-1960", "start_display": None, "end": "01-1970", "end_display": None }, { "key": None, "start": None, "start_display": None, "end": "now-10M/M", "end_display": None } ] }, { "type": "range", "field": "metadata.budget", "negated": False, "must": True, "id": "budget_r", "ui_only": { "display_name": "Budget Range", "hide_count": False, "hide_if_one": False, "translate": None }, "aggs": None, "ranges": [ { "key": "< 10,000,000", "start": None, "end": "10000000" }, { "key": "> 10,000,000", "start": "10000000", "end": None }, { "key": "50 to 100 millions", "start": "50000000", "end": "100000000" } ], "multi_select": False } ], "type": "DynamicAggStage", "enable": True, "name": "aggregations", "save_to_intermediate": False, "expand_result": False, "ui_only": None, "filters_name": "aggregations_filters", 'engine_name': DEFAULT_ENGINE_NAME, "search_response": "search", "ui_aggregations": True }, { "type": "FilterStage", "enable": True, "name": "filters", "engine_name": DEFAULT_ENGINE_NAME, "save_to_intermediate": False, "expand_result": False, "filters": [ { "type": "term", "field": "adult", "negated": False, "must": True, "value": False, "multi_select": False } ] }, { 'type': 'QueryStage', 'engine_name': DEFAULT_ENGINE_NAME, 'enable': True, 'qpl_enable': True, 'name': 'search', 'index': 'movies', 'fields': ['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], 'range_fields': ['metadata.release_date'], 'fetch_fields': ['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], 'implicit_operator': 'or', 'aggregations': 'aggregations', 'aggregations_filters': 'aggregations_filters', 'ui_only': { 'sort': { 'default': { 'field': '_score', 'display_name': 'Score', 'order': 'desc' }, 'options': [ { 'field': '_score', 'display_name': 'Score', 'order': 'desc' }, { 'field': 'release_date', 'display_name': 'Release Date', 'order': 'asc' }, { 'field': 'metadata.budget', 'display_name': 'Budget', 'order': 'desc' } ] }, 'page_size': { 'default': 25, 'options': [25, 50, 100] } }, 'sort': { 'field': '_score', 'order': 'desc' }, 'page_size_default': 25, 'highlight_stage': 'highlight' }, { 'type': 'DynamicResultsStage', 'engine_name': DEFAULT_ENGINE_NAME, 'enable': True, 'name': 'hits', 'title': { 'type': 'url', 'field': 'title', 'link': 'url' }, 'sub_title': { 'type': 'text', 'field': 'url', }, 'thumbnail': None, 'non_displayed_fields': [], 'header': [], 'body': [ [{'type': 'bool', 'label': 'Adult', 'field': 'adult'}, {'type': 'number', 'label': 'Popularity', 'field': 'metadata.popularity', 'format': '1.2'} ], [{'type': 'text', 'field': 'overview'}], [{'type': 'text', 'label': 'Status', 'field': 'status'}, {'type': 'date', 'label': 'Release Date', 'field': 'release_date', 'format': '%Y-%m-%d'}], [{'type': 'thumbnails', 'field': 'metadata.cast[*].profile_path_w185', 'alt': 'metadata.cast[*].name', 'detail': 'metadata.cast[*].name', 'full_size': 'metadata.cast[*].profile_path' }] ], 'metadata': { 'General': { 'fields': [ { 'type': 'chip', 'label': 'Chip', 'field': 'metadata.genres.name', 'unique': True }, {'type': 'text', 'field': 'title', 'label': 'Title'} ], 'sort': 'asc' }, }, 'search_response': 'search' }, ] } |
Code Block | ||||||||||
---|---|---|---|---|---|---|---|---|---|---|
| ||||||||||
from app.pipeline import Pipeline from app.pipeline.stages import HighlightStage, DynamicAggStage, FilterStage, \ DynamicResultsStage, QueryStage from app.pipeline.stages.query import QueryStageUIConfig from framework.aggregations import * from framework.fields import * from framework.filters import * from models.engines import HighlightField, SortConfig, PageSizeConfig, SortEntry from models.utils import SortOrder from utils.constants import DEFAULT_ENGINE_NAME SEARCH_STAGE_NAME = 'search' ######################################################################################################################## # PIPELINE STRUCTURE ######################################################################################################################## PIPELINE = Pipeline( enable=True, stages=[ HighlightStage( enable=True, name='highlight', engine_name=DEFAULT_ENGINE_NAME, pre_tags=['<strong>'], pos_tags=['</strong>'], tags_schema=None, force_source=False, require_field_match=True, number_of_fragments=3, fragment_size=100, order=None, fields=[ HighlightField(field='title', number_of_fragments=1), HighlightField(field='overview', number_of_fragments=5), HighlightField(field='url', number_of_fragments=1), ] ), FilterStage( enable=True, name='filters', engine_name=DEFAULT_ENGINE_NAME, save_to_intermediate=False, expand_result=False, filters=[ TermFilter(field='adult', negated=False, must=True, value=False, multi_select=False) ] ), DynamicAggStage( enable=True, name='aggregations', filters_name='aggregations_filters', save_to_intermediate=False, expand_result=False, engine_name=DEFAULT_ENGINE_NAME, search_response=SEARCH_STAGE_NAME, ui_aggregations=True, # This over here generates the aggregations for ui, if you don't want them, set it to False aggs=[ TermAgg( id='genres', field='metadata.genres.name.keyword', must=True, max_values=100, multi_select=True, operation='or', order=None, exclude_terms=None, include_terms=None, ui_only=TermAggUI(display_name='Genres', max_display=5, expanded=True), aggs=[ TopHitsAgg( id='top_hits', type='top_hits' ), TermAgg( field='metadata.spoken_languages.name.keyword', negated=False, must=True, id='spoken_languages', ui_only=TermAggUI(display_name='Spoken Languages', max_display=5, expanded=False), aggs=None, max_values=None, multi_select=False, operation='or', order=None, exclude_terms=None, include_terms=None )] ), BoolAgg( id='adult', field='adult', negated=False, must=True, ui_only=BoolAggUI(display_name='Adult', display_True='True', display_False='False') ), DateHistogramAgg( field='release_date', negated=False, must=True, id='release_date_h', ui_only=DateHistogramAggUI(display_name='Release Date'), calendar_interval=['1y', '1q', '1M', '1d'], format=['yyyy', '\'Q\'q', 'MMMM', 'E dd'], time_zone=['-06:00'], min_doc_count=[100, 1], offset=None, order=None ), HistogramAgg( id='budget', field='metadata.budget', negated=False, must=True, interval=[10000000, 1000000], min_doc_count=[100, 10, 1], order=None, offset=None, ui_only=HistogramAggUI(display_name='Budget') ), SliderAgg( id='budget_slider', field='metadata.budget', negated=False, must=True, ui_only=SliderAggUI(display_name='Budget', translate=None, min=None, max=None, show_ticks=True), ), DateRangeAgg( field='release_date', negated=False, must=True, id='release_date_dr', ui_only=DateRangeAggUI( display_name='Release Range', hide_count=False, translate=None ), format='MM-yyyy', ranges=[ DateRange( key='minus_10', start='now-10y', start_display=None, end=None, end_display=None ), DateRange( key=None, start='01-1960', start_display=None, end='01-1970', end_display=None ), DateRange( key=None, start=None, start_display=None, end='now-10M/M', end_display=None ) ] ), RangeAgg( field='metadata.budget', negated=False, must=True, id='budget_r', multi_select=False, ui_only=RangeAggUI( display_name='Budget Range', hide_count=False, translate=None ), ranges=[ Range( key='< 10,000,000', start=None, end='10000000' ), Range( key='> 10,000,000', start='10000000', end=None ), Range( key='50 to 100 millions', start='50000000', end='100000000' ) ], ) ] ), DynamicResultsStage( engine_name=DEFAULT_ENGINE_NAME, enable=True, name='hits', title=UrlField(field='title', link='url'), sub_title=TextField(field='url'), thumbnail=None, non_displayed_fields=[], body=[ [BoolField(label='Adult', field='adult'), NumberField(label='Popularity', field='metadata.popularity', format='1.2')], [TextField(field='overview')], [TextField(label='Status', field='status'), DateField(label='Release Date', field='release_date', format='%Y-%m-%d')], [ThumbnailsField(field='metadata.cast[*].profile_path_w185', alt='metadata.cast[*].name', detail='metadata.cast[*].name', full_size='metadata.cast[*].profile_path')] ], search_response=SEARCH_STAGE_NAME ), QueryStage( engine_name=DEFAULT_ENGINE_NAME, enable=True, qpl_enable=True, name=SEARCH_STAGE_NAME, index='movies', wildcard=False, fields=['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], range_fields=['metadata.budget'], date_fields=['release_date'], fetch_fields=['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], exclude_fields=None, implicit_operator='or', page_size_default=25, sort=SortEntry( field='_score', order=SortOrder.DESC ), aggregations='aggregations', aggregations_filters='aggregations_filters', highlight='highlight_stage', filters='filters', ui_only=QueryStageUIConfig( sort=SortConfig( default=SortEntry( field='_score', display_name='Score', order=SortOrder.DESC ), options=[ SortEntry( field='_score', display_name='Score', order=SortOrder.DESC ), SortEntry( field='release_date', display_name='Release Date', order=SortOrder.ASC ), SortEntry( field='metadata.budget', display_name='Budget', order=SortOrder.DESC ) ] ), page_size=PageSizeConfig( default=25, options=[25, 50, 100] ) ) ) ] ) |
Code Block | ||||||||||
---|---|---|---|---|---|---|---|---|---|---|
| ||||||||||
from app.pipeline import Pipeline from app.pipeline.stages import HighlightStage, DynamicAggStage, FilterStage, \ DynamicResultsStage, QueryStage from app.pipeline.stages.query import QueryStageUIConfig from framework.aggregations import * from framework.fields import * from framework.filters import * from models.engines import HighlightField, SortConfig, PageSizeConfig, SortEntry from models.utils import SortOrder from utils.constants import DEFAULT_ENGINE_NAME ######################################################################################################################## # STAGES CONFIGURATION ######################################################################################################################## SEARCH_STAGE_NAME = 'search' ################# # HIGHLIGHT ################# _highlight_stage = HighlightStage( enable=True, name='highlight', engine_name=DEFAULT_ENGINE_NAME, pre_tags=['<strong>'], pos_tags=['</strong>'], tags_schema=None, force_source=False, require_field_match=True, number_of_fragments=3, fragment_size=100, order=None, fields=[ HighlightField(field='title', number_of_fragments=1), HighlightField(field='overview', number_of_fragments=5), HighlightField(field='url', number_of_fragments=1), ] ) ################# # AGGREGATIONS ################# _aggregation_stage = DynamicAggStage( enable=True, name='aggregations', filters_name='aggregations_filters', save_to_intermediate=False, expand_result=False, engine_name=DEFAULT_ENGINE_NAME, search_response=SEARCH_STAGE_NAME, ui_aggregations=True, # This over here generates the aggregations for ui, if you don't want them, set it to False aggs=[ TermAgg( id='genres', field='metadata.genres.name.keyword', must=True, max_values=100, multi_select=True, operation='or', order=None, exclude_terms=None, include_terms=None, ui_only=TermAggUI(display_name='Genres', max_display=5, expanded=True), aggs=[ TopHitsAgg( id='top_hits', type='top_hits' ), TermAgg( field='metadata.spoken_languages.name.keyword', negated=False, must=True, id='spoken_languages', ui_only=TermAggUI(display_name='Spoken Languages', max_display=5, expanded=False), aggs=None, max_values=None, multi_select=False, operation='or', order=None, exclude_terms=None, include_terms=None )] ), BoolAgg( id='adult', field='adult', negated=False, must=True, ui_only=BoolAggUI(display_name='Adult', display_True='True', display_False='False') ), DateHistogramAgg( field='release_date', negated=False, must=True, id='release_date_h', ui_only=DateHistogramAggUI(display_name='Release Date'), calendar_interval=['1y', '1q', '1M', '1d'], format=['yyyy', '\'Q\'q', 'MMMM', 'E dd'], time_zone=['-06:00'], min_doc_count=[100, 1], offset=None, order=None ), HistogramAgg( id='budget', field='metadata.budget', negated=False, must=True, interval=[10000000, 1000000], min_doc_count=[100, 10, 1], order=None, offset=None, ui_only=HistogramAggUI(display_name='Budget') ), SliderAgg( id='budget_slider', field='metadata.budget', negated=False, must=True, ui_only=SliderAggUI(display_name='Budget', translate=None, min=None, max=None, show_ticks=True), ), DateRangeAgg( field='release_date', negated=False, must=True, id='release_date_dr', ui_only=DateRangeAggUI( display_name='Release Range', hide_count=False, translate=None ), format='MM-yyyy', ranges=[ DateRange( key='minus_10', start='now-10y', start_display=None, end=None, end_display=None ), DateRange( key=None, start='01-1960', start_display=None, end='01-1970', end_display=None ), DateRange( key=None, start=None, start_display=None, end='now-10M/M', end_display=None ) ] ), RangeAgg( field='metadata.budget', negated=False, must=True, id='budget_r', multi_select=False, ui_only=RangeAggUI( display_name='Budget Range', hide_count=False, translate=None ), ranges=[ Range(key='< 10,000,000', start=None, end='10000000'), Range(key='> 10,000,000', start='10000000', end=None), Range(key='50 to 100 millions', start='50000000', end='100000000') ], ) ] ) ################# # FILTER ################# _filter_stage = FilterStage( enable=True, name='filters', engine_name=DEFAULT_ENGINE_NAME, save_to_intermediate=False, expand_result=False, filters=[ TermFilter(field='adult', negated=False, must=True, value=False, multi_select=False) ] ) ################# # UI RESULTS ################# _dynamic_results = DynamicResultsStage( engine_name=DEFAULT_ENGINE_NAME, enable=True, name='hits', title=UrlField(field='title', link='url'), sub_title=TextField(field='url'), thumbnail=None, non_displayed_fields=[], body=[ [BoolField(label='Adult', field='adult'), NumberField(label='Popularity', field='metadata.popularity', format='1.2')], [TextField(field='overview')], [TextField(label='Status', field='status'), DateField(label='Release Date', field='release_date', format='%Y-%m-%d')], [ThumbnailsField(field='metadata.cast[*].profile_path_w185', alt='metadata.cast[*].name', detail='metadata.cast[*].name', full_size='metadata.cast[*].profile_path')] ], search_response=SEARCH_STAGE_NAME ) ################# # SEARCH ################# _query_stage = QueryStage( engine_name=DEFAULT_ENGINE_NAME, enable=True, qpl_enable=True, name=SEARCH_STAGE_NAME, index='movies', wildcard=False, fields=['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], range_fields=['metadata.budget'], date_fields=['release_date'], fetch_fields=['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata', 'metadata.production_companies', 'metadata.cast', 'metadata.directors'], exclude_fields=None, implicit_operator='or', page_size_default=25, sort=SortEntry( field='_score', order=SortOrder.DESC ), aggregations=_aggregation_stage.name, aggregations_filters=_aggregation_stage.filters_name, highlight=_highlight_stage.name, filters=_filter_stage.name, ui_only=QueryStageUIConfig( sort=SortConfig( default=SortEntry( field='_score', display_name='Score', order=SortOrder.DESC ), options=[ SortEntry( field='_score', display_name='Score', order=SortOrder.DESC ), SortEntry( field='release_date', display_name='Release Date', order=SortOrder.ASC ), SortEntry( field='metadata.budget', display_name='Budget', order=SortOrder.DESC ) ] ), page_size=PageSizeConfig( default=25, options=[25, 50, 100] ) ) ) ######################################################################################################################## # PIPELINE STRUCTURE ######################################################################################################################## PIPELINE = Pipeline( enable=True, stages=[ _highlight_stage, _filter_stage, _aggregation_stage, _dynamic_results, _query_stage ] ) |
1) Create the pipeline file
Start by creating a python file for the pipeline configuration in the folder config/pipeline
Tip |
---|
A descriptive name works the best |
On The File
Code Block | ||||
---|---|---|---|---|
| ||||
from app.pipeline import Pipeline from app.pipeline.stages import DemoStage |
Code Block | ||||
---|---|---|---|---|
| ||||
_demo = DemoStage( enable=True, name='demo', save_to_intermediate=False, expand_result=False, ui_only=None, halt_on_exception=False ) |
Code Block | ||||
---|---|---|---|---|
| ||||
PIPELINE = Pipeline( enable=True, stages=[ _demo ] ) |
You can go to /es/docs and execute the GET pipeline endpoint and see if the name appears