← Back
Editing: robotparser.cpython-35.opt-2.pyc
���]4 � @ s^ d d l Z d d l Z d g Z Gd d � d � Z Gd d � d � Z Gd d � d � Z d S)� N�RobotFileParserc @ s e Z d Z d d d � Z d d � Z d d � Z d d � Z d d � Z d d � Z d d � Z d d � Z d d � Z d S)r � c C s> g | _ d | _ d | _ d | _ | j | � d | _ d S)NFr )�entries� default_entry� disallow_all� allow_all�set_url� last_checked)�self�url� r �0/opt/alt/python35/lib64/python3.5/robotparser.py�__init__ s zRobotFileParser.__init__c C s | j S)N)r )r r r r �mtime s zRobotFileParser.mtimec C s d d l } | j � | _ d S)Nr )�timer )r r r r r �modified( s zRobotFileParser.modifiedc C s5 | | _ t j j | � d d � \ | _ | _ d S)N� � )r �urllib�parse�urlparse�host�path)r r r r r r 0 s zRobotFileParser.set_urlc C s� y t j j | j � } Wnm t j j k r� } zG | j d k rO d | _ n' | j d k rv | j d k rv d | _ WYd d } ~ Xn) X| j � } | j | j d � j � � d S)N� � Ti� i� zutf-8)r r ) r Zrequest�urlopenr �error� HTTPError�coder r �readr �decode� splitlines)r �f�err�rawr r r r 5 s zRobotFileParser.readc C s> d | j k r* | j d k r: | | _ n | j j | � d S)N�*)� useragentsr r �append)r �entryr r r � _add_entryB s zRobotFileParser._add_entryc C s d } t � } | j � x�| D]�} | sr | d k rJ t � } d } n( | d k rr | j | � t � } d } | j d � } | d k r� | d | � } | j � } | s� q | j d d � } t | � d k r | d j � j � | d <t j j | d j � � | d <| d d k r_| d k rB| j | � t � } | j j | d � d } q | d d k r�| d k r�| j j t | d d � � d } q | d d k r | d k r | j j t | d d � � d } q W| d k r�| j | � d S) Nr r � �#�:z user-agentZdisallowFZallowT)�Entryr r) �find�strip�split�len�lowerr r �unquoter& r' � rulelines�RuleLine)r �lines�stater( �line�ir r r r K sL zRobotFileParser.parsec C s� | j r d S| j r d S| j s' d St j j t j j | � � } t j j d d | j | j | j | j f � } t j j | � } | s� d } x- | j D]" } | j | � r� | j | � Sq� W| j r� | j j | � Sd S)NFTr �/)r r r r r r r3 � urlunparser �paramsZqueryZfragment�quoter � applies_to� allowancer )r � useragentr Z parsed_urlr( r r r � can_fetch s$ zRobotFileParser.can_fetchc C s d j d d � | j D� � S)Nr c S s g | ] } t | � d � q S)� )�str)�.0r( r r r � <listcomp>� s z+RobotFileParser.__str__.<locals>.<listcomp>)�joinr )r r r r �__str__� s zRobotFileParser.__str__N) �__name__� __module__� __qualname__r r r r r r) r rA rG r r r r r s 4 c @ s4 e Z d Z d d � Z d d � Z d d � Z d S)r5 c C sY | d k r | r d } t j j t j j | � � } t j j | � | _ | | _ d S)Nr T)r r r; r r= r r? )r r r? r r r r � s zRuleLine.__init__c C s | j d k p | j | j � S)Nr% )r � startswith)r �filenamer r r r> � s zRuleLine.applies_toc C s | j r d n d d | j S)NZAllowZDisallowz: )r? r )r r r r rG � s zRuleLine.__str__N)rH rI rJ r r> rG r r r r r5 � s r5 c @ s@ e Z d Z d d � Z d d � Z d d � Z d d � Z d S) r- c C s g | _ g | _ d S)N)r& r4 )r r r r r � s zEntry.__init__c C sj g } x' | j D] } | j d | d g � q Wx* | j D] } | j t | � d g � q: Wd j | � S)Nz User-agent: rB r )r& �extendr4 rC rF )r �ret�agentr8 r r r rG � s z Entry.__str__c C s] | j d � d j � } x= | j D]2 } | d k r9 d S| j � } | | k r# d Sq# Wd S)Nr: r r% TF)r0 r2 r& )r r@ rO r r r r> � s zEntry.applies_toc C s. x' | j D] } | j | � r | j Sq Wd S)NT)r4 r> r? )r rL r8 r r r r? � s zEntry.allowanceN)rH rI rJ r rG r> r? r r r r r- � s r- )Z urllib.parser Zurllib.request�__all__r r5 r- r r r r �<module> s �
Save File
Cancel