- Created by Potter (Esteban Alvarado), last modified on Jun 08, 2023
You are viewing an old version of this page. View the current version.
Compare with Current View Page History
« Previous Version 4 Next »
Pipeline Definition
All Pipeline:
- Are .py files, meaning they are programmable
- Declare a PIPELINE variable of type Pipeline, or Dictionary if you are going to use the JSON format (which we do not recommend if possible)
Dictionary
Intended for automatic generation, web UI, and Automation NOT for manual pipeline building
Don't use this format
Pipeline
Intended manual usage in IDE, all settings should have a class for either Dictionary or Pipeline format, so why not use the configuration class directly
Pipeline With Variables
No difference with previuos one, regarding functionality, but much more manageable when defining, and modifying
Pros
- Easier to read
- Generic JSON format
Cons
- Requires conversion to Python classes
- Verbose
- All in one single JSON object
- Susceptible to conversion and format issues
Pros
- No conversion need
- Easier for IDE (Specially Pycharm with Pydantic Plug-in)
- Type errors, in pipeline definition capture via IDE compiler or in runtime easily
- Can go directly to class if need to check types or other parameters
Cons
- You may need to import a lot of configuration classes
- A bit harder to read
Pros
- … Same as previuos one..
- Each stage is an independent variable, allowing to use its parameters, in following stages
- Better comments
- Pipline structure simplified,
and easier to modify
Cons
- You may need to import a lot of configuration classes
- A bit harder to read
- Verbose
JSON Format Expand source
from utils.constants import DEFAULT_ENGINE_NAME
PIPELINE = {
'enable': True,
'stages': [
{
"type": "HighlightStage",
"enable": True,
"name": "highlight",
"save_to_intermediate": False,
"expand_result": False,
"ui_only": None,
'engine_name': DEFAULT_ENGINE_NAME,
"pre_tags": [
"<strong>"
],
"pos_tags": [
"</strong>"
],
"tags_schema": None,
"force_source": False,
"require_field_match": True,
"number_of_fragments": 100,
"fragment_size": 100,
"order": None,
"fields": [
{
"field": "title",
"fragment_size": None,
"number_of_fragments": 1,
"highlight_query": None,
"type": None,
"force_source": False
},
{
"field": "overview",
"fragment_size": None,
"number_of_fragments": 5,
"highlight_query": None,
"type": None,
"force_source": False
},
{
"field": "url",
"fragment_size": None,
"number_of_fragments": 1,
"highlight_query": None,
"type": None,
"force_source": False
}
]
},
{
"aggs": [
{
"type": "term",
"field": "metadata.genres.name.keyword",
"negated": False,
"must": True,
"id": "genres",
"ui_only": {
"display_name": "Genres",
"hide_count": False,
"hide_if_one": False,
"translate": None,
"max_display": 5,
"mime_icon": False,
"expanded": True,
"enable_negative": True
},
"aggs": [
{
"id": "top_hits",
"type": "top_hits"
},
{
"type": "term",
"field": "metadata.spoken_languages.name.keyword",
"negated": False,
"must": True,
"id": "spoken_languages",
"ui_only": {
"display_name": "Spoken Languages",
"hide_count": False,
"hide_if_one": False,
"translate": None,
"max_display": 5,
"mime_icon": False,
"expanded": False,
"enable_negative": False
},
"aggs": None,
"max_values": None,
"multi_select": False,
"operation": "or",
"order": None,
"exclude_terms": None,
"include_terms": None
}],
"max_values": 100,
"multi_select": True,
"operation": "or",
"order": None,
"exclude_terms": None,
"include_terms": None
},
{
"type": "bool",
"field": "adult",
"negated": False,
"must": True,
"id": "adult",
"ui_only": {
"display_name": "Adult",
"hide_count": False,
"hide_if_one": False,
"translate": None,
"display_True": "True",
"display_False": "False"
},
"aggs": None
},
{
"type": "date_histogram",
"field": "release_date",
"negated": False,
"must": True,
"id": "release_date_h",
"ui_only": {
"display_name": "Release Date",
"hide_if_one": False,
"hide_count": False,
"translate": None,
"level": 0
},
"aggs": None,
"calendar_interval": ["1y", "1q", "1M", "1d"],
"format": ["yyyy", "'Q'q", "MMMM", "E dd"],
"time_zone": ["-06:00"],
"min_doc_count": [100, 1],
"offset": None,
"order": None
},
{
"type": "histogram",
"field": "metadata.budget",
"negated": False,
"must": True,
"id": "budget",
"ui_only": {
"display_name": "Budget",
"hide_if_one": False,
"hide_count": False,
"translate": None,
"level": 0
},
"aggs": None,
"interval": [10000000, 1000000],
"min_doc_count": [100, 10, 1],
"order": None,
"offset": None
},
{
"type": "slider",
"field": "metadata.budget",
"negated": False,
"must": True,
"id": "budget_slider",
"ui_only": {
"display_name": "Budget",
"hide_count": False,
"hide_if_one": False,
"translate": None,
"min": None,
"max": None,
"show_ticks": True
},
"aggs": None
},
{
"type": "date_range",
"field": "release_date",
"negated": False,
"must": True,
"id": "release_date_dr",
"ui_only": {
"display_name": "Release Range",
"hide_count": False,
"hide_if_one": False,
"translate": None
},
"aggs": None,
"format": "MM-yyyy",
"ranges": [
{
"key": "minus_10",
"start": "now-10y",
"start_display": None,
"end": None,
"end_display": None
},
{
"key": None,
"start": "01-1960",
"start_display": None,
"end": "01-1970",
"end_display": None
},
{
"key": None,
"start": None,
"start_display": None,
"end": "now-10M/M",
"end_display": None
}
]
},
{
"type": "range",
"field": "metadata.budget",
"negated": False,
"must": True,
"id": "budget_r",
"ui_only": {
"display_name": "Budget Range",
"hide_count": False,
"hide_if_one": False,
"translate": None
},
"aggs": None,
"ranges": [
{
"key": "< 10,000,000",
"start": None,
"end": "10000000"
},
{
"key": "> 10,000,000",
"start": "10000000",
"end": None
},
{
"key": "50 to 100 millions",
"start": "50000000",
"end": "100000000"
}
],
"multi_select": False
}
],
"type": "DynamicAggStage",
"enable": True,
"name": "aggregations",
"save_to_intermediate": False,
"expand_result": False,
"ui_only": None,
"filters_name": "aggregations_filters",
'engine_name': DEFAULT_ENGINE_NAME,
"search_response": "search",
"ui_aggregations": True
},
{
"type": "FilterStage",
"enable": True,
"name": "filters",
"engine_name": DEFAULT_ENGINE_NAME,
"save_to_intermediate": False,
"expand_result": False,
"filters": [
{
"type": "term",
"field": "adult",
"negated": False,
"must": True,
"value": False,
"multi_select": False
}
]
},
{
'type': 'QueryStage',
'engine_name': DEFAULT_ENGINE_NAME,
'enable': True,
'qpl_enable': True,
'name': 'search',
'index': 'movies',
'fields': ['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies',
'metadata.cast',
'metadata.directors'],
'range_fields': ['metadata.release_date'],
'fetch_fields': ['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata',
'metadata.production_companies', 'metadata.cast', 'metadata.directors'],
'implicit_operator': 'or',
'aggregations': 'aggregations',
'aggregations_filters': 'aggregations_filters',
'ui_only': {
'sort': {
'default': {
'field': '_score',
'display_name': 'Score',
'order': 'desc'
},
'options': [
{
'field': '_score',
'display_name': 'Score',
'order': 'desc'
},
{
'field': 'release_date',
'display_name': 'Release Date',
'order': 'asc'
},
{
'field': 'metadata.budget',
'display_name': 'Budget',
'order': 'desc'
}
]
},
'page_size': {
'default': 25,
'options': [25, 50, 100]
}
},
'sort': {
'field': '_score',
'order': 'desc'
},
'page_size_default': 25,
'highlight_stage': 'highlight'
},
{
'type': 'DynamicResultsStage',
'engine_name': DEFAULT_ENGINE_NAME,
'enable': True,
'name': 'hits',
'title': {
'type': 'url',
'field': 'title',
'link': 'url'
},
'sub_title': {
'type': 'text',
'field': 'url',
},
'thumbnail': None,
'non_displayed_fields': [],
'header': [],
'body': [
[{'type': 'bool', 'label': 'Adult', 'field': 'adult'},
{'type': 'number', 'label': 'Popularity', 'field': 'metadata.popularity',
'format': '1.2'}
],
[{'type': 'text', 'field': 'overview'}],
[{'type': 'text', 'label': 'Status', 'field': 'status'},
{'type': 'date', 'label': 'Release Date', 'field': 'release_date',
'format': '%Y-%m-%d'}],
[{'type': 'thumbnails', 'field': 'metadata.cast[*].profile_path_w185',
'alt': 'metadata.cast[*].name', 'detail': 'metadata.cast[*].name',
'full_size': 'metadata.cast[*].profile_path'
}]
],
'metadata': {
'General': {
'fields': [
{
'type': 'chip', 'label': 'Chip', 'field': 'metadata.genres.name',
'unique': True
},
{'type': 'text', 'field': 'title', 'label': 'Title'}
],
'sort': 'asc'
},
},
'search_response': 'search'
},
]
}
Python Format Expand source
from app.pipeline import Pipeline
from app.pipeline.stages import HighlightStage, DynamicAggStage, FilterStage, \
DynamicResultsStage, QueryStage
from app.pipeline.stages.query import QueryStageUIConfig
from framework.aggregations import *
from framework.fields import *
from framework.filters import *
from models.engines import HighlightField, SortConfig, PageSizeConfig, SortEntry
from models.utils import SortOrder
from utils.constants import DEFAULT_ENGINE_NAME
SEARCH_STAGE_NAME = 'search'
########################################################################################################################
# PIPELINE STRUCTURE
########################################################################################################################
PIPELINE = Pipeline(
enable=True,
stages=[
HighlightStage(
enable=True,
name='highlight',
engine_name=DEFAULT_ENGINE_NAME,
pre_tags=['<strong>'],
pos_tags=['</strong>'],
tags_schema=None,
force_source=False,
require_field_match=True,
number_of_fragments=3,
fragment_size=100,
order=None,
fields=[
HighlightField(field='title', number_of_fragments=1),
HighlightField(field='overview', number_of_fragments=5),
HighlightField(field='url', number_of_fragments=1),
]
),
FilterStage(
enable=True,
name='filters',
engine_name=DEFAULT_ENGINE_NAME,
save_to_intermediate=False,
expand_result=False,
filters=[
TermFilter(field='adult', negated=False, must=True, value=False, multi_select=False)
]
),
DynamicAggStage(
enable=True,
name='aggregations',
filters_name='aggregations_filters',
save_to_intermediate=False,
expand_result=False,
engine_name=DEFAULT_ENGINE_NAME,
search_response=SEARCH_STAGE_NAME,
ui_aggregations=True,
# This over here generates the aggregations for ui, if you don't want them, set it to False
aggs=[
TermAgg(
id='genres',
field='metadata.genres.name.keyword',
must=True,
max_values=100,
multi_select=True,
operation='or',
order=None,
exclude_terms=None,
include_terms=None,
ui_only=TermAggUI(display_name='Genres', max_display=5, expanded=True),
aggs=[
TopHitsAgg(
id='top_hits',
type='top_hits'
),
TermAgg(
field='metadata.spoken_languages.name.keyword',
negated=False,
must=True,
id='spoken_languages',
ui_only=TermAggUI(display_name='Spoken Languages', max_display=5, expanded=False),
aggs=None,
max_values=None,
multi_select=False,
operation='or',
order=None,
exclude_terms=None,
include_terms=None
)]
),
BoolAgg(
id='adult',
field='adult',
negated=False,
must=True,
ui_only=BoolAggUI(display_name='Adult', display_True='True', display_False='False')
),
DateHistogramAgg(
field='release_date',
negated=False,
must=True,
id='release_date_h',
ui_only=DateHistogramAggUI(display_name='Release Date'),
calendar_interval=['1y', '1q', '1M', '1d'],
format=['yyyy', '\'Q\'q', 'MMMM', 'E dd'],
time_zone=['-06:00'],
min_doc_count=[100, 1],
offset=None,
order=None
),
HistogramAgg(
id='budget',
field='metadata.budget',
negated=False,
must=True,
interval=[10000000, 1000000],
min_doc_count=[100, 10, 1],
order=None,
offset=None,
ui_only=HistogramAggUI(display_name='Budget')
),
SliderAgg(
id='budget_slider',
field='metadata.budget',
negated=False,
must=True,
ui_only=SliderAggUI(display_name='Budget', translate=None, min=None, max=None, show_ticks=True),
),
DateRangeAgg(
field='release_date',
negated=False,
must=True,
id='release_date_dr',
ui_only=DateRangeAggUI(
display_name='Release Range',
hide_count=False,
translate=None
),
format='MM-yyyy',
ranges=[
DateRange(
key='minus_10',
start='now-10y',
start_display=None,
end=None,
end_display=None
),
DateRange(
key=None,
start='01-1960',
start_display=None,
end='01-1970',
end_display=None
),
DateRange(
key=None,
start=None,
start_display=None,
end='now-10M/M',
end_display=None
)
]
),
RangeAgg(
field='metadata.budget',
negated=False,
must=True,
id='budget_r',
multi_select=False,
ui_only=RangeAggUI(
display_name='Budget Range',
hide_count=False,
translate=None
),
ranges=[
Range(
key='< 10,000,000',
start=None,
end='10000000'
),
Range(
key='> 10,000,000',
start='10000000',
end=None
),
Range(
key='50 to 100 millions',
start='50000000',
end='100000000'
)
],
)
]
),
DynamicResultsStage(
engine_name=DEFAULT_ENGINE_NAME,
enable=True,
name='hits',
title=UrlField(field='title', link='url'),
sub_title=TextField(field='url'),
thumbnail=None,
non_displayed_fields=[],
body=[
[BoolField(label='Adult', field='adult'),
NumberField(label='Popularity', field='metadata.popularity', format='1.2')],
[TextField(field='overview')],
[TextField(label='Status', field='status'),
DateField(label='Release Date', field='release_date', format='%Y-%m-%d')],
[ThumbnailsField(field='metadata.cast[*].profile_path_w185', alt='metadata.cast[*].name',
detail='metadata.cast[*].name', full_size='metadata.cast[*].profile_path')]
],
search_response=SEARCH_STAGE_NAME
),
QueryStage(
engine_name=DEFAULT_ENGINE_NAME,
enable=True,
qpl_enable=True,
name=SEARCH_STAGE_NAME,
index='movies',
wildcard=False,
fields=['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies',
'metadata.cast',
'metadata.directors'],
range_fields=['metadata.budget'],
date_fields=['release_date'],
fetch_fields=['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata',
'metadata.production_companies', 'metadata.cast', 'metadata.directors'],
exclude_fields=None,
implicit_operator='or',
page_size_default=25,
sort=SortEntry(
field='_score',
order=SortOrder.DESC
),
aggregations='aggregations',
aggregations_filters='aggregations_filters',
highlight='highlight_stage',
filters='filters',
ui_only=QueryStageUIConfig(
sort=SortConfig(
default=SortEntry(
field='_score',
display_name='Score',
order=SortOrder.DESC
),
options=[
SortEntry(
field='_score',
display_name='Score',
order=SortOrder.DESC
),
SortEntry(
field='release_date',
display_name='Release Date',
order=SortOrder.ASC
),
SortEntry(
field='metadata.budget',
display_name='Budget',
order=SortOrder.DESC
)
]
),
page_size=PageSizeConfig(
default=25,
options=[25, 50, 100]
)
)
)
]
)
Python with Variables Expand source
from app.pipeline import Pipeline
from app.pipeline.stages import HighlightStage, DynamicAggStage, FilterStage, \
DynamicResultsStage, QueryStage
from app.pipeline.stages.query import QueryStageUIConfig
from framework.aggregations import *
from framework.fields import *
from framework.filters import *
from models.engines import HighlightField, SortConfig, PageSizeConfig, SortEntry
from models.utils import SortOrder
from utils.constants import DEFAULT_ENGINE_NAME
########################################################################################################################
# STAGES CONFIGURATION
########################################################################################################################
SEARCH_STAGE_NAME = 'search'
#################
# HIGHLIGHT
#################
_highlight_stage = HighlightStage(
enable=True,
name='highlight',
engine_name=DEFAULT_ENGINE_NAME,
pre_tags=['<strong>'],
pos_tags=['</strong>'],
tags_schema=None,
force_source=False,
require_field_match=True,
number_of_fragments=3,
fragment_size=100,
order=None,
fields=[
HighlightField(field='title', number_of_fragments=1),
HighlightField(field='overview', number_of_fragments=5),
HighlightField(field='url', number_of_fragments=1),
]
)
#################
# AGGREGATIONS
#################
_aggregation_stage = DynamicAggStage(
enable=True,
name='aggregations',
filters_name='aggregations_filters',
save_to_intermediate=False,
expand_result=False,
engine_name=DEFAULT_ENGINE_NAME,
search_response=SEARCH_STAGE_NAME,
ui_aggregations=True, # This over here generates the aggregations for ui, if you don't want them, set it to False
aggs=[
TermAgg(
id='genres',
field='metadata.genres.name.keyword',
must=True,
max_values=100,
multi_select=True,
operation='or',
order=None,
exclude_terms=None,
include_terms=None,
ui_only=TermAggUI(display_name='Genres', max_display=5, expanded=True),
aggs=[
TopHitsAgg(
id='top_hits',
type='top_hits'
),
TermAgg(
field='metadata.spoken_languages.name.keyword',
negated=False,
must=True,
id='spoken_languages',
ui_only=TermAggUI(display_name='Spoken Languages', max_display=5, expanded=False),
aggs=None,
max_values=None,
multi_select=False,
operation='or',
order=None,
exclude_terms=None,
include_terms=None
)]
),
BoolAgg(
id='adult',
field='adult',
negated=False,
must=True,
ui_only=BoolAggUI(display_name='Adult', display_True='True', display_False='False')
),
DateHistogramAgg(
field='release_date',
negated=False,
must=True,
id='release_date_h',
ui_only=DateHistogramAggUI(display_name='Release Date'),
calendar_interval=['1y', '1q', '1M', '1d'],
format=['yyyy', '\'Q\'q', 'MMMM', 'E dd'],
time_zone=['-06:00'],
min_doc_count=[100, 1],
offset=None,
order=None
),
HistogramAgg(
id='budget',
field='metadata.budget',
negated=False,
must=True,
interval=[10000000, 1000000],
min_doc_count=[100, 10, 1],
order=None,
offset=None,
ui_only=HistogramAggUI(display_name='Budget')
),
SliderAgg(
id='budget_slider',
field='metadata.budget',
negated=False,
must=True,
ui_only=SliderAggUI(display_name='Budget', translate=None, min=None, max=None, show_ticks=True),
),
DateRangeAgg(
field='release_date',
negated=False,
must=True,
id='release_date_dr',
ui_only=DateRangeAggUI(
display_name='Release Range',
hide_count=False,
translate=None
),
format='MM-yyyy',
ranges=[
DateRange(
key='minus_10',
start='now-10y',
start_display=None,
end=None,
end_display=None
),
DateRange(
key=None,
start='01-1960',
start_display=None,
end='01-1970',
end_display=None
),
DateRange(
key=None,
start=None,
start_display=None,
end='now-10M/M',
end_display=None
)
]
),
RangeAgg(
field='metadata.budget',
negated=False,
must=True,
id='budget_r',
multi_select=False,
ui_only=RangeAggUI(
display_name='Budget Range',
hide_count=False,
translate=None
),
ranges=[
Range(key='< 10,000,000', start=None, end='10000000'),
Range(key='> 10,000,000', start='10000000', end=None),
Range(key='50 to 100 millions', start='50000000', end='100000000')
],
)
]
)
#################
# FILTER
#################
_filter_stage = FilterStage(
enable=True,
name='filters',
engine_name=DEFAULT_ENGINE_NAME,
save_to_intermediate=False,
expand_result=False,
filters=[
TermFilter(field='adult', negated=False, must=True, value=False, multi_select=False)
]
)
#################
# UI RESULTS
#################
_dynamic_results = DynamicResultsStage(
engine_name=DEFAULT_ENGINE_NAME,
enable=True,
name='hits',
title=UrlField(field='title', link='url'),
sub_title=TextField(field='url'),
thumbnail=None,
non_displayed_fields=[],
body=[
[BoolField(label='Adult', field='adult'),
NumberField(label='Popularity', field='metadata.popularity', format='1.2')],
[TextField(field='overview')],
[TextField(label='Status', field='status'),
DateField(label='Release Date', field='release_date', format='%Y-%m-%d')],
[ThumbnailsField(field='metadata.cast[*].profile_path_w185', alt='metadata.cast[*].name',
detail='metadata.cast[*].name', full_size='metadata.cast[*].profile_path')]
],
search_response=SEARCH_STAGE_NAME
)
#################
# SEARCH
#################
_query_stage = QueryStage(
engine_name=DEFAULT_ENGINE_NAME,
enable=True,
qpl_enable=True,
name=SEARCH_STAGE_NAME,
index='movies',
wildcard=False,
fields=['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies',
'metadata.cast',
'metadata.directors'],
range_fields=['metadata.budget'],
date_fields=['release_date'],
fetch_fields=['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata',
'metadata.production_companies', 'metadata.cast', 'metadata.directors'],
exclude_fields=None,
implicit_operator='or',
page_size_default=25,
sort=SortEntry(
field='_score',
order=SortOrder.DESC
),
aggregations=_aggregation_stage.name,
aggregations_filters=_aggregation_stage.filters_name,
highlight=_highlight_stage.name,
filters=_filter_stage.name,
ui_only=QueryStageUIConfig(
sort=SortConfig(
default=SortEntry(
field='_score',
display_name='Score',
order=SortOrder.DESC
),
options=[
SortEntry(
field='_score',
display_name='Score',
order=SortOrder.DESC
),
SortEntry(
field='release_date',
display_name='Release Date',
order=SortOrder.ASC
),
SortEntry(
field='metadata.budget',
display_name='Budget',
order=SortOrder.DESC
)
]
),
page_size=PageSizeConfig(
default=25,
options=[25, 50, 100]
)
)
)
########################################################################################################################
# PIPELINE STRUCTURE
########################################################################################################################
PIPELINE = Pipeline(
enable=True,
stages=[
_highlight_stage,
_filter_stage,
_aggregation_stage,
_dynamic_results,
_query_stage
]
)
- No labels
Overview
Content Tools
- Powered by Atlassian Confluence 7.19.22
- Printed by Atlassian Confluence 7.19.22
- Report a bug
- Atlassian News