HabitSystemV3: ethiopian_spiderling_config.py

#===============================================================================
#   SpiderLing -- A web spider/crawler for linguistic purpose.
#   Copyright (c) 2011-2017  Vit Suchomel
#   http://corpus.tools/wiki/SpiderLing
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
#===============================================================================

#===============================================================================
# SPIDERLING CONFIGURATION FILE
# Set the most important configuration options here
# Sections: general, connection, domains, processing, debug
#===============================================================================


#== general ==

#file directories
RUN_DIR = u'run'
LOG_DIR = RUN_DIR
PIPE_DIR = u'%s/pipes' % RUN_DIR #interprocess communication
URL_DIR = u'%s/url' % RUN_DIR #URLs to download
ROBOTS_DIR = u'%s/robots' % RUN_DIR #robots.txt
REDIRECT_DIR = u'%s/redirect' % RUN_DIR #HTTP redirections
WPAGE_DIR = u'%s/wpage' % RUN_DIR #raw encoded downloaded documents
ARC_DIR = u'%s/arc' % RUN_DIR #raw encoded downloaded documents in the arc format
DOC_META_DIR = u'%s/docmeta' % RUN_DIR #document metadata and extracted URLs
PREVERT_DIR = u'%s/prevert' % RUN_DIR #processed documents in the prevertical format
IGNORED_DIR = u'%s/ignored' % RUN_DIR #URLs excluded from further processing
SAVE_DIR = u'%s/save' % RUN_DIR #saved states

#max crawling time [s] (None for no limit)
MAX_RUN_TIME = None

#The number of downloaded data processing threads (process.py) to run,
#2 is enough for very small languages, 16 is enough for huge English crawls.
#Do not set to more than (<number of your processors> - 2).
#A single document processor can deal with approx. 1000 MB/hour raw data
#download speed (a rough estimate, depends on CPU).
DOC_PROCESSOR_COUNT = 4

#Interpreter and priority of subprocesses opened by the main script
DOWNLOADER_EXEC = ['python', 'crawl.py']
PROCESSOR_EXEC = ['ionice', '-c3', 'nice', '-n10', 'python', 'process.py']


#== connection ==

#You are responsible for properly identifying your crawler.
#Some sites use the '+' in AGENT_URL to reject bots => it is polite to keep it.
#The crawler's behaviour and possible masking is __your__ responsibility.
AGENT = 'SpiderLing'
AGENT_URL = '+http://nlp.fi.muni.cz/projects/biwec/'
USER_AGENT = 'Mozilla/5.0 (compatible; SpiderLing (a SPIDER for LINGustic research); %s)' % AGENT_URL
#Ignore the robots protocol if the crawler failed to fetch or parse robots.txt.
IGNORE_ROBOTS_WHEN_FAILED = True #False
#Give up waiting for robots.txt for a domain in time [s] (None to turn off).
#Apply the ignore decision (see above).
MAX_ROBOT_WAITING_TIME = 3*3600

#https support, fall back to http if disabled
#Used ssl.SSLContext functions require Python >= 2.7.9.
#Disabling results in ignoring webs requiring SSL.
SSL_ENABLED = True

#http response size constraints
DOC_SIZE_MIN = 200
DOC_SIZE_MAX = 10*1024*1024 #do not set to less than 10000

#max number of open connections (max: ulimit -n), must be >> OPEN_AT_ONCE
MAX_OPEN_CONNS = 8000
#max number of connections opened at once
OPEN_AT_ONCE = 800
#abandon open connection when not responding after time [s]
NO_ACTIVITY_TIMEOUT = 80 #40

#max number of urls waiting to be downloaded, decrease to save the RAM
MAX_URL_QUEUE = 5000000
#max number of documents waiting to be sent
MAX_WPAGE_QUEUE = 5000

#period [s] of connecting to the same IP
IP_CONN_INTERVAL = 1
#period [s] of connecting to the same host =~ crawl delay (20 means 3/min)
HOST_CONN_INTERVAL = 20
#The crawler's behaviour is __your__ responsibility, 
#setting these values too low will result in banning your IP by target sites
#and can lead to blocking your internet connection by your ISP!


#== domains ==

#min docs downloaded from a domain before yield rate threshold applies
MIN_DOCS_DOWNLOADED = 6
#min bytes downloaded from a domain before yield rate threshold applies
MIN_BYTES_DOWNLOADED = 256*1024
#max (soft limit) cleaned non-empty docs from a domain (None for no limit)
MAX_DOCS_CLEANED = None

"""
The yield rate threshold function sets the threshold to stop crawling a domain
when it is no more efficient.
"""
from math import log
def yield_rate_threshold(doc_count):
    #standard threshold: 10: 0 %, 100: 1 %, 1000: 2 %, 10000: 3 %, 100000: 4 %
    #return 0.01 * (log(doc_count, 10) - 1)
    return 0.001 * (log(doc_count, 10) - 1) #small languages
    #return 0.0 #threshold disabled

#The web domain data (paths within the domain) is the most memory consuming
#structure. A rough estimate is 500 MB - 5 GB per million domains, it depends
#on the number of urls within a stored domain. The more domains the crawler has
#to store, the more memory allocated. After reaching MAX_DOMS_READY domains,
#additional domains in state "ready" are written into domains_oversize file
#and never examined again. Therefore do not set MAX_DOMS_READY to a low value.
#Domains in state "new" (IP not determined yet) consume memory in the same way.
#The size of that queue is not limited. Therefore your free memory can be
#depleted by "new" domains despite you limited "ready" domains.

#max number of domains ready for crawling kept in memory
#1M is enough for small languages (expect ~2 GB RAM required),
#50M should be enough for huge English crawls (allocates up to ~200 GB RAM).
MAX_DOMS_READY = 5*1000*1000
#max domains selected for crawling at the same time
MAX_DOMS_SELECTED = 500*1000
#max time [s] no data received from a selected domain
MAX_IDLE_TIME = 6*3600 #1800
#period [s] of domain refresh (adding new, removing old, evaluating efficiency)
UPDATE_DOMAINS_PERIOD = 600
#max number of domains with the same IP selected for crawling at the same time
MAX_IP_REPEAT = 50

#Number of urls sent at once by the scheduler to the crawler. Decrease the
#minimum value to 10 if crawling a small language. Set the minimum starting
#value to 1 if crawling a single web domain. Increase the maximum value to
#allow wider domain coverage in the downloader's url queue.
#Must be < MAX_DOMS_SELECTED.
MIN_URL_SELECT_START = 100
MIN_URL_SELECT = 1000
MAX_URL_SELECT = 100000
#Decrease to improve domain variety and thus crawling speed, useful only when
#there is more than ~500 domains crawled concurrently (big languages)
MAX_URL_SELECT_PER_DOMAIN = 100

#hostname -> IP mapping file (two space separated columns), None by default
#Use recent DNS mappings only since hosts tend to move occasionally.
DNS_TABLE_PATH = None #u'url/dns_table' #None


#== processing ==

#Set the languages of documents to recognise, e.g. ['Norwegian', 'Danish'].
#This enables recognising unwanted languages similar to target languages.
#Requires
# - a plaintext in that languages: ./util/lang_samples/Norwegian
# - a jusText wordlist for that languages: ./util/justext_wordlists/Norwegian
# - a chared model for that languages: ./util/chared_models/Norwegian
#The most similar language is selected (if below lang diff thresholds).
LANGUAGES = ['Amharic', 'Oromo', 'Somali', 'Tigrinya', 'English', 'Arabic']
#Set which of recognised languages to accept, e.g. ['Norwegian'].
LANGUAGES_ACCEPT = ['Amharic', 'Oromo', 'Somali', 'Tigrinya']
#max allowed difference to the lang model for document and paragraphs
#(0.0 = same as the model); it does not work well with paragraphs, the reported
#similarity is usually low
LANG_DIFF_THRESHOLD_DOC = 0.7
LANG_DIFF_THRESHOLD_PAR = 0.99
#disable encoding detection and force an encoding (e.g. 'utf-8'), None by default
FORCE_ENCODING = None
#switch to unigram models (useful for Chinese, Japanese, Korean)
UNIGRAM_MODELS = False

#Allowed top level domains regexp -- crawling is restricted to these, e.g.:
#English + general web: u'\.(?:uk|us|au|ca|com|org|net|info|biz|edu|name)$'
#German + general web: u'\.(?:de|at|ch|com|org|net|info|eu|edu|name)$'
#Czech only: u'\.cz$'
#no restriction (all): u''
from re import compile
TLD_WHITELIST_RE = compile(u'')
#Country/language native TLDs, e.g. u'\.(?:uk|us|au|ca)$' or u'\.cz$' or u''.
#If EXTRACT_EMPTY_PAGE_EXTERNAL_LINKS == True, a matched cross domain link
#will be followed. u'' is the default and results in following all links.
TLD_NATIVE_RE = compile(u'\.(?:et|er|so|dj)$')
#Use e.g. compile(u'\.da$') to block the Danish TLD,
#TLD blacklist > TLD whitelist, None by default.
TLD_BLACKLIST_RE = None #compile(u'')

#file containing unwanted web domains (one per line), None by default
DOMAIN_BLACKLIST_PATH = u'url/domain_blacklist' #None
#file containing allowed web domains (one per line), None by default
#priority: blacklist > whitelist > allowed TLDs
DOMAIN_WHITELIST_PATH = None

#extract links from web pages containing no clean text
#useful to decrease crawling from domains in unwanted languages
#which is dealt with by the trigram model
#disabling this may increase crawling efficiency (yield rate)
#at the cost of omitting some web domains (or visiting them later)
EXTRACT_EMPTY_PAGE_INTERNAL_LINKS = True
#extract links from pages with no clean text going out from the domain
#priority: EXTRACT_EMPTY_PAGE_LINKS > EXTRACT_EMPTY_PAGE_EXTERNAL_LINKS
EXTRACT_EMPTY_PAGE_EXTERNAL_LINKS = False

#justext paragraph heuristic configuration
#character count < length_low => bad or short
JUSTEXT_LENGTH_LOW = 50 #justext default = 70
#character count > length_high => good
JUSTEXT_LENGTH_HIGH = 100 #justext default = 200
#number of words frequent in the language >= stopwords_low => neargood
JUSTEXT_STOPWORDS_LOW = 0.2 #justext default = 0.3
#number of words frequent in the language >= stopwords_high => good or neargood
JUSTEXT_STOPWORDS_HIGH = 0.3 #justext default = 0.32
#density of link words (words inside the <a> tag) > max_link_density => bad
JUSTEXT_MAX_LINK_DENSITY = 0.4 #justext default = 0.2
#short/near-good headings in the distance of <value> characters before a good paragraph => good
JUSTEXT_MAX_HEADING_DISTANCE = 200 #justext default = 200

#path to justext wordlists, use None for the justext default
#JUSTEXT_WORDLIST_DIR/LANGUAGE must exist for all LANGUAGES
JUSTEXT_WORDLIST_DIR = u'util/justext_wordlists'
#path to chared models
#CHARED_MODEL_DIR/LANGUAGE must exist for all LANGUAGES
CHARED_MODEL_DIR = u'util/chared_models'
#allow conversion of binary formats (pdf, doc) to text (not tested,
#the conversion is not reliable, may result in the presence of garbage text
CONVERSION_ENABLED = False


#== debug ==

import logging
LOG_LEVEL = logging.DEBUG #DEBUG|INFO|WARNING|ERROR
LOG_FORMAT = u'%(asctime)-15s %(threadName)s %(message)s'
#period [s] of writing out some debug info, 0 to disable
INFO_PERIOD = 3600
#period [s] of saving state of the crawler (all domain data), 0 to disable
SAVE_PERIOD = 3600*24
#log stream buffering (0 = none, 1 = line, -1 = fully/system, n = n bytes)
LOG_BUFFERING = 1