KaneAI - World’s First E2E Software Testing Agent.Book a Demo Join TestMu Conference

How to use elasticsearch method in localstack

Best Python code snippet using localstack_python

elasticsearch_operator.py

Source:elasticsearch_operator.py

...24        super().__init__()25        self.operator_data_counter = 026        self.target_system_config = SystemConfig.get_config(is_source=False)27        if self.configs.operator_source == Constants.elasticsearch:28            self.elasticsearch = ElasticsearchHook.get_elasticsearch(29                hosts=list(map((lambda x: x["host"]), SystemConfig.get_config(is_source=True)["hostList"])),30                index=self.configs.elasticsearch_index31            )32    def accumulate_data(self, elasticsearch):33        data_to_be_written = []34        search_result, scroll_size, scroll_id = self.operations.operation(self.configs.operator_source, "search")(35            elasticsearch,36            self.configs.elasticsearch_index,37            self.configs.elasticsearch_query,38            self.configs.elasticsearch_search_limit,39            self.configs.elasticsearch_scroll)40        while scroll_size > 0:41            for s3_data in search_result['hits']['hits']:42                self.operator_data_counter += 1...

__main__.py

Source:__main__.py

1import json2from pulumi import Config, ResourceOptions3from pulumi.stack_reference import StackReference4from pulumi_aws import ec2, iam, s35from ol_infrastructure.lib.aws.ec2_helper import (6    DiskTypes,7    InstanceTypes,8    build_userdata,9    debian_10_ami,10    default_egress_args,11)12from ol_infrastructure.lib.aws.iam_helper import lint_iam_policy13from ol_infrastructure.lib.ol_types import AWSBase14from ol_infrastructure.lib.pulumi_helper import parse_stack15from ol_infrastructure.providers.salt.minion import (16    OLSaltStackMinion,17    OLSaltStackMinionInputs,18)19stack_info = parse_stack()20env_config = Config("environment")21elasticsearch_config = Config("elasticsearch")22salt_config = Config("saltstack")23environment_name = f"{stack_info.env_prefix}-{stack_info.env_suffix}"24business_unit = env_config.get("business_unit") or "operations"25aws_config = AWSBase(tags={"OU": business_unit, "Environment": environment_name})26network_stack = StackReference(f"infrastructure.aws.network.{stack_info.name}")27consul_stack = StackReference(28    "infrastructure.consul."29    f"{stack_info.namespace.rsplit('.', 1)[1]}."30    f"{stack_info.name}"31)32destination_vpc = network_stack.require_output(env_config.require("vpc_reference"))33elasticsearch_backup_bucket_name = f"ol-{environment_name}-elasticsearch-backup"34elasticsearch_backup_bucket = s3.Bucket(35    f"ol-{environment_name}-elasticsearch-backup",36    bucket=elasticsearch_backup_bucket_name,37    acl="private",38    tags=aws_config.tags,39    versioning={"enabled": True},40    server_side_encryption_configuration={41        "rule": {42            "applyServerSideEncryptionByDefault": {43                "sseAlgorithm": "aws:kms",44            },45        },46    },47)48elasticsearch_instance_policy = {49    "Version": "2012-10-17",50    "Statement": [51        {52            "Action": [53                "ec2:DescribeInstances",54                "ec2:DescribeAvailabilityZones",55                "ec2:DescribeRegions",56                "ec2:DescribeSecurityGroups",57                "ec2:DescribeTags",58            ],59            "Effect": "Allow",60            "Resource": ["*"],61        },62        {63            "Action": [64                "s3:AbortMultipartUpload",65                "s3:ListBucket",66                "s3:GetObject*",67                "s3:PutObject",68                "s3:DeleteObject",69            ],70            "Effect": "Allow",71            "Resource": [72                f"arn:aws:s3:::{elasticsearch_backup_bucket_name}",73                f"arn:aws:s3:::{elasticsearch_backup_bucket_name}/*",74            ],75        },76    ],77}78elasticsearch_iam_policy = iam.Policy(79    f"elasticsearch-policy-{environment_name}",80    name=f"elasticsearch-policy-{environment_name}",81    path=f"/ol-applications/elasticsearch-{environment_name}/",82    policy=lint_iam_policy(elasticsearch_instance_policy, stringify=True),83    description="Policy to access resources needed by elasticsearch instances",84)85elasticsearch_instance_role = iam.Role(86    "elasticsearch-instance-role",87    assume_role_policy=json.dumps(88        {89            "Version": "2012-10-17",90            "Statement": {91                "Effect": "Allow",92                "Action": "sts:AssumeRole",93                "Principal": {"Service": "ec2.amazonaws.com"},94            },95        }96    ),97    name=f"elasticsearch-instance-role-{environment_name}",98    path=f"/ol-operations/elasticsearch-{environment_name}/",99    tags=aws_config.tags,100)101iam.RolePolicyAttachment(102    f"elasticsearch-role-policy-{environment_name}",103    policy_arn=elasticsearch_iam_policy.arn,104    role=elasticsearch_instance_role.name,105)106elasticsearch_instance_profile = iam.InstanceProfile(107    f"elasticsearch-instance-profile-{environment_name}",108    role=elasticsearch_instance_role.name,109    name=f"elasticsearch-instance-profile-{environment_name}",110    path="/ol-operations/elasticsearch-profile/",111)112elasticsearch_security_group = ec2.SecurityGroup(113    f"elasticsearch-{environment_name}",114    name=f"elasticsearch-{environment_name}",115    description="Access control between Elasticsearch instances in cluster",116    tags=aws_config.merged_tags({"Name": f"{environment_name}-elasticsearch"}),117    vpc_id=destination_vpc["id"],118    ingress=[119        ec2.SecurityGroupIngressArgs(120            protocol="tcp",121            from_port=9200,122            to_port=9200,123            cidr_blocks=[destination_vpc["cidr"]],124            description="Access to Elasticsearch cluster from VPC",125        ),126        ec2.SecurityGroupIngressArgs(127            self=True,128            protocol="tcp",129            from_port=9300,130            to_port=9400,131            description="Elasticsearch cluster instances access",132        ),133    ],134    egress=default_egress_args,135)136security_groups = {"elasticsearch_server": elasticsearch_security_group.id}137instance_type_name = (138    elasticsearch_config.get("instance_type") or InstanceTypes.burstable_medium.name139)140instance_type = InstanceTypes[instance_type_name].value141elasticsearch_instances = []142export_data = {}143subnets = destination_vpc["subnet_ids"]144salt_environment = Config("saltstack").get("environment_name") or environment_name145instance_nums = range(elasticsearch_config.get_int("instance_count") or 3)146for instance_num in instance_nums:147    instance_name = f"elasticsearch-{environment_name}-{instance_num}"148    salt_minion = OLSaltStackMinion(149        f"saltstack-minion-{instance_name}",150        OLSaltStackMinionInputs(minion_id=instance_name),151    )152    cloud_init_userdata = build_userdata(153        instance_name=instance_name,154        minion_keys=salt_minion,155        minion_roles=["elasticsearch"],156        minion_environment=salt_environment,157        salt_host=f"salt-{stack_info.env_suffix}.private.odl.mit.edu",158        additional_cloud_config={159            "fs_setup": [160                {161                    "device": "/dev/nvme0n1",162                    "filesystem": "ext4",163                    "label": "var_lib_elastics",164                    "partition": "None",165                }166            ],167            "mounts": [168                [169                    "LABEL=var_lib_elastics",170                    "/var/lib/elasticsearch",171                    "ext4",172                    "defaults",173                    "0",174                    "0",175                ]176            ],177        },178    )179    instance_tags = aws_config.merged_tags(180        {181            "Name": instance_name,182            "elasticsearch_env": environment_name,183            "escluster": salt_environment,184        }185    )186    es_security_groups = [187        destination_vpc["security_groups"]["salt_minion"],188        consul_stack.require_output("security_groups")["consul_agent"],189        elasticsearch_security_group.id,190    ]191    if elasticsearch_config.get_bool("public_web"):192        es_security_groups.append(destination_vpc["security_groups"]["web"])193    elasticsearch_instance = ec2.Instance(194        f"elasticsearch-instance-{environment_name}-{instance_num}",195        ami=elasticsearch_config.get("ami_id") or debian_10_ami.id,196        user_data=cloud_init_userdata,197        instance_type=instance_type,198        iam_instance_profile=elasticsearch_instance_profile.id,199        tags=instance_tags,200        volume_tags=instance_tags,201        subnet_id=subnets[instance_num],202        key_name=salt_config.require("key_name"),203        root_block_device=ec2.InstanceRootBlockDeviceArgs(204            volume_type=DiskTypes.ssd,205            volume_size=20,206            encrypted=True,207        ),208        ebs_block_devices=[209            ec2.InstanceEbsBlockDeviceArgs(210                device_name="/dev/sdb",211                volume_type=DiskTypes.ssd,212                volume_size=elasticsearch_config.get_int("disk_size") or 100,213                encrypted=True,214            )215        ],216        vpc_security_group_ids=es_security_groups,217        opts=ResourceOptions(depends_on=[salt_minion]),218    )219    elasticsearch_instances.append(elasticsearch_instance)220    export_data[instance_name] = {221        "public_ip": elasticsearch_instance.public_ip,222        "private_ip": elasticsearch_instance.private_ip,223        "ipv6_address": elasticsearch_instance.ipv6_addresses,...

elasticsearch_return.py

Source:elasticsearch_return.py

1#! -*- coding: utf-8 -*-2'''3Return data to an elasticsearch server for indexing.4:maintainer:    Jurnell Cockhren <jurnell.cockhren@sophicware.com>5:maturity:      New6:depends:       `elasticsearch-py <http://elasticsearch-py.readthedocs.org/en/latest/>`_,  `jsonpickle <https://pypi.python.org/pypi/jsonpickle>`_7:platform:      all8To enable this returner the elasticsearch python client must be installed9on the desired minions (all or some subset).10The required configuration is as follows:11.. code-block:: yaml12    elasticsearch:13      host: 'somehost.example.com:9200'14      index: 'salt'15      number_of_shards: 1 (optional)16      number_of_replicas: 0 (optional)17or to specify multiple elasticsearch hosts for resiliency:18.. code-block:: yaml19    elasticsearch:20      host:21        - 'somehost.example.com:9200'22        - 'anotherhost.example.com:9200'23        - 'yetanotherhost.example.com:9200'24      index: 'salt'25      number_of_shards: 1 (optional)26      number_of_replicas: 0 (optional)27The above configuration can be placed in a targeted pillar, minion or28master configurations.29To use the returner per salt call:30.. code-block:: bash31    salt '*' test.ping --return elasticsearch32In order to have the returner apply to all minions:33.. code-block:: yaml34    ext_job_cache: elasticsearch35'''36from __future__ import absolute_import37# Import Python libs38import datetime39# Import Salt libs40import salt.utils.jid41__virtualname__ = 'elasticsearch'42try:43    import elasticsearch44    HAS_ELASTICSEARCH = True45except ImportError:46    HAS_ELASTICSEARCH = False47try:48    from jsonpickle.pickler import Pickler49    HAS_PICKLER = True50except ImportError:51    HAS_PICKLER = False52def _create_index(client, index):53    '''54    Create empty index55    '''56    client.indices.create(57        index=index,58        body={59            'settings': {60                'number_of_shards': __salt__['config.get']('elasticsearch:number_of_shards') or 1,61                'number_of_replicas': __salt__['config.get']('elasticsearch:number_of_replicas') or 0,62            },63            'mappings': {64                'returner': {65                    'properties': {66                        '@timestamp': {67                            'type': 'date'68                        },69                        'success': {70                            'type': 'boolean'71                        },72                        'id': {73                            'type': 'string'74                        },75                        'retcode': {76                            'type': 'integer'77                        },78                        'fun': {79                            'type': 'string'80                        },81                        'jid': {82                            'type': 'string'83                        }84                    }85                }86            }87        },88        ignore=40089    )90def __virtual__():91    if HAS_ELASTICSEARCH and HAS_PICKLER:92        return __virtualname__93    return False94def _get_pickler():95    '''96    Return a picker instance97    '''98    return Pickler(max_depth=5)99def _get_instance():100    '''101    Return the elasticsearch instance102    '''103    # Check whether we have a single elasticsearch host string, or a list of host strings.104    if isinstance(__salt__['config.get']('elasticsearch:host'), list):105        return elasticsearch.Elasticsearch(__salt__['config.get']('elasticsearch:host'))106    else:107        return elasticsearch.Elasticsearch([__salt__['config.get']('elasticsearch:host')])108def returner(ret):109    '''110    Process the return from Salt111    '''112    es_ = _get_instance()113    _create_index(es_, __salt__['config.get']('elasticsearch:index'))114    the_time = datetime.datetime.now().isoformat()115    ret['@timestamp'] = the_time116    es_.index(index=__salt__['config.get']('elasticsearch:index'),117             doc_type='returner',118             body=_get_pickler().flatten(ret),119             )120def prep_jid(nocache=False, passed_jid=None):  # pylint: disable=unused-argument121    '''122    Do any work necessary to prepare a JID, including sending a custom id123    '''...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.