1
일부 파일을 AWS ES로 구문 분석하려고하는 Python 코드가 있습니다. 아래 코드는 다음과 같습니다Elasticsearch TransportError (400, 'mapper_parsing_exception')
TransportError(400, 'mapper_parsing_exception')
이 인덱스 라인에서 발생
import re
import elasticsearch
import itertools
from elasticsearch import Elasticsearch, RequestsHttpConnection, helpers
from boto3 import client, logging, s3, Session
access_key = 'accesskey'
secret_key = 'secretkey'
bucket = 'bucket'
pattern = '(\d{4}[-]\d{2}[-]\d{2})\s+(\d{2}[:]\d{2}[:]\d{2})\s+(\d{1,3}[.]\d{1,3}[.]\d{1,3}[.]\d{1,3})\s+([GET|PUT|POST]+)\s+([^\s]+)\s+(\d+)\s+(\d+)\s+(\d+)\s+([^\s]+)\s+(\".*?\")\s+(\".*?\")'
folder = 'subfolder/'
host = 'host'
index_name = 'test_index'
es = Elasticsearch(
hosts = host,
connection_class = RequestsHttpConnection,
port = 443,
use_ssl = True,
verify_certs = False)
request_body = {
'settings': {
'number_of_shards': 5,
'number_of_relicas': 0
},
'mappings': {
'logs': {
'properties': {
'date': { 'type': 'date', 'format': 'YYYY-MM-DD' },
'time': { 'type': 'string' },
'client_ip': { 'type': 'ip' },
'method': { 'type': 'string' },
'request': { 'type': 'string' },
'status_code': { 'type': 'integer' },
'bytes': { 'type': 'integer' },
'time_taken': { 'type': 'integer' },
'referrer': { 'type': 'string' },
'user_agent': { 'type': 'string' },
'cookie': { 'type': 'string' }
}
}
}
}
if es.indices.exists('test_index'):
print(index_name + 'Exists')
else:
res = es.indices.create(index = index_name, ignore = [400, 404], body = request_body, request_timeout = 30)
session = Session(
aws_access_key_id = access_key,
aws_secret_access_key = secret_key,
region_name = 'region')
s3 = session.resource('s3')
bucket_name = s3.Bucket(bucket)
data = [obj for obj in list(bucket_name.objects.filter(Prefix=folder)) if obj.key != folder]
bulk_data = []
for obj in data:
body = obj.get()['Body'].read().decode('ISO-8859-1')
match = re.compile(pattern)
for line in body.splitlines():
if not line.startswith('#'):
result = match.search(line)
x = 1
if result is not None:
for i in range(11):
es.index(index = index_name, doc_type = 'logs', body = result.group(x))
x += 1
그러나, 나는 오류가 계속. 데이터를 확인했는데 데이터가 올바르게 읽혀지고있는 것처럼 보입니다. 다음은 색인을 생성하려는 문서의 예입니다.
#Version: 1.0
#Fields: date time cs-ip cs-method cs-uri sc-status sc-bytes time-taken cs(Referer) cs(User-Agent) cs(Cookie)
2016-06-14 18:20:07 1.1.1.1 GET /origin.com/products/PublishingImages/file.png 200 10915 1 "http://www.referrer.com/CategoryPage.aspx?" "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0)" "ASP.NET_SessionId=yjumie0ar12dl; __SessionCookie=7FKbuqkC6pmz2ZKewLVMAjOtKTbLpb8zK8j0TCr+U; __CSALastPageName=XCOaCvdhusg==; __CSALastHierarchy=L7ATLEuRZmXj3ddzXRPF7BVVZFzs=; __CSALastReportSuiteName=0eWICfQzEtcnfMSQ==; __CSARedirectTags=ABOcOdYBWK/O5Rw==; pck=0aa46f231-abc9-cc8d12f4b2d9; __CSCookie=RVqW7dLcpb4QVYDUPoADt5vqtK0oo9QKniVXlY0O83Cet4n73Fah5UvdTzbfayAEgnHsTch/OrCOSTZaK4peWy/1kXq3fX2uHLPbrw4BgXhaZvjRUp4yRhaL+Ug2qcj6pQny6oyot+GPOBP4OgqtbzTZJv7NxykKZHKZRzjYBvBLtbOf9brrzjzeaTbDw7r7SEmVzxQOihmDBGDG0Rn16jC9T8MtbUl3VKz3GojZeLTmYTCXX67oBmnGZPPtV8joGqYrVcHrSj0XKxyeFBluQONFGvEGhw/7TKoshKQfd5oNo7A9eJ+UfEvG51iwT/3ePMd6MvzFMdj+eMuizFcjdas9v+bWH923AYt7rUnb99T8AX6+8cJy0/fj69AkdhG/FTvNQrV05Lpbr5gjL5uy8N4Ng2zsoKMdgrFYalARQNM/8h0i7HhxQyx+OY6PXR0VbTGI=; ETE1=fvClmAi4/MuJaZFB+0Cac7U1k4DgF5ylbBWLfPA5UuABTPG9Xp+KAVSMf3ES0cJm94ok4ZwIQ1Y/AiaX1j7lf6BQpcoI9UwGAe9PUFnpSnxiAlp9I5fSbCJyho3YDa0MoJ9xC7fwwPigfrd7oxdyi8nvCKs5OYcrf0zEAMiMUEMFpaB//Imb3d015qESq0aThMWwhuKwT7C7xyk4Q1mdD0l3kFum4C1g2cbIaNQ5qMJ3M+DJvu7SbjVIAZanGwrt1aZLL/1Eo1MPVEuGmMOj/p6hHtjBc; icxid=1461781512418-308716; gigyaCookieNotification=acknowledged; __PublicSiteRecentSearchesCookie=UeUbHRveUtzmA==; __CSUserIbcIDCookie=dHEe9zrGnQX6OLzOhtqfft1DC571R+670C7m6; __CSUserIbcMonikerCookie=Oiz4bVglhbAzQ==; __qca=P0-455562-1461759; icxid=146512418-3087371016; s_vi=[CS]v1|2B90051D1FEE-4000196D[CE]; s_fid=0C6E0CDA143CF-181DECF8985; ddtLatC=549p52.5p46.5p37p41p55.5p35p105; dtCookie=B66FBB0F22CBB49B863198A2"
2016-06-14 18:20:07 1.1.1.1 GET /origin.com/UI/Core/Images/2c.png 304 204 0 "http://www.referrer.com/CategoryPage.aspx?" "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0)" "ASP.NET_SessionId=yjumie0d5l12dl; __SessionCookie=7FKbuq3luPyu/Tz2ZKewLVMAjOtKTbLpb8zK8j0TCr+U; __CSALastPageName=XCOaFcpXdhusg==; __CSALastHierarchy=L7ATLYBSXZmXj3ddzXRPF7BVVZFzs=; __CSALastReportSuiteName=0eWIC8dcnfMSQ==; __CSARedirectTags=ABOcOxeO5Rw==; pck=0aa46f23-cc8d12f4b2d9; __CSCookie=RVqW7dLcpCsk3wIlGmvZSVdPmNjkdGJ9+zMtfOYdMV/fQBUDLKlF01gkDTUg9IENJq+mVzOR1wbxACRCgIcBWYnPY0O83Cet4n73Fah5UvdTzbfayAEgnHsTch/OrCOSTZaK4peWy/1kXq3fX2uHLPbrw4BgXhaZvjRUp4yRhaL+Ug2qcj6pQny6oyot+GPOBP4OgqtbzTZJv7NxykKZHKZRzjYBvBLtbOf9brrzjOihmDBGDG0Rn16jC9T8MtbUl3VKz3GojZeLTmYTCXX67oBmnGZPPtV8joGqYrVcHrSaSmLvVeYt9ieNaiJRkSNnqZMHN0oj0XKxyeFBluQONFGvEGhw/7TKoshKQfd5oNo7A9eJ+UfEvG51iwT/3ePMd6MvzFMdj+5Fg6MIlBuFYXrwVOwIeuhFcjdas9v+bWH923AYt7rUnb99T8AX6+8cJy0/fj69AkdhG/FTvNQrV05Lpbr5gjL5uy8N4Ng2zsoKMdgrFYalARQNM/8h0i7HhxQyx+OY6PXR0VbTGI=; ETE1=fvClmAi4/MuJaZFB+0Cac7U1k4DPA5UuABTPG9Xp+KAVSMf3ES0cJm94ok4ZwIQ1Y/AiaX1j7lf6BQpcoI9UwGAe9PUFnpSn3YDa0MoJ9xC7fwwPigfrd7oxdyi8nvCKs5OYcrf0zEAMiMUEM//Imb3d015qESq0aThM4W51AmI4EEDdFk2l8aqkDjAE8ZmWwhuKwT7C7xyk4Q1mdD0l3kFum4C1g2cbIaNQ5qMJ3M+DJvu7SbjVIAZanGwrt1aZLL/1Eo1MPVEuGmMOj/p6hHtjBc; icxid=1461418-3071016; gigyaCookieNotification=acknowledged; __PublicSiteRecentSearchesCookie=UeUbHUqUtzmA==; __CSUserIbcIDCookie=dHEe9zrPGNScOhtqfft1DC571R+670C7m6; __CSUserIbcMonikerCookie=Oiz4bVglGrhbAzQ==; __qca=P0-455503562-1461259; icxid=14612418-30873371016; s_vi=[CS]v1|2B917051D1FEE-40001000FA6D[CE]; s_fid=0C6EA0A143CF-1810DF8985; dtPC=-; s_cc=true; dtLatC=54p52.5p46.5p37p4p41p55.5p3p105; dtCookie=B66FBB0F863198A2D45|VVMr1BXU3wx"
내가 뭘 잘못하고 있는지 알 수 있습니까?
편집 : 전체 오류가 캡처 됨. 여기에 출력됩니다
{"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"failed to p
arse"}],"type":"mapper_parsing_exception","reason":"failed to parse","caused_by"
:{"type":"not_x_content_exception","reason":"Compressor detection can only be ca
lled on some xcontent bytes or compressed xcontent bytes"}},"status":400
작은 따옴표가있는 본문 이외에는 어떤 문제도 표시되지 않습니다. –
작은 따옴표로 인해 문제가 발생합니까? – vagenheart
실제로 문제가 될지 확실하지 않습니다. 하지만 request_body를 복사하고 나머지 콘솔에서 시도해도 request_body를 구문 분석 할 수 없었습니다. 큰 따옴표로 변경하면 효과가있었습니다. 그게 여기서 잘 될지 모르겠습니다. –