B
    0*\k                 @   s  d Z ddlZddlZddlmZ ddlZddlmZ ddlZddl	m
Z
 ddlmZ ddlZddlZddlmZ ddlZddlZe ZdaG d	d
 d
Zdd ZdZG dd deZG dd dZdd ZG dd deZG dd deZG dd deZ G dd deZ!dd Z"dd Z#d0d d!Z$d"d# Z%d$d% Z&d&d' Z'da(da)d(d) Z*d*d+ Z+G d,d- d-ej,Z-G d.d/ d/ej.Z/e0e dS )1z"Brian Quinlan (brian@sweetapp.com)    N)_base)Full)wait)Queue)partialFc               @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
_ThreadWakeupc             C   s   t jdd\| _| _d S )NF)Zduplex)mpZPipe_reader_writer)self r   EC:\ALexclude\prg\programme\Python37\Lib\concurrent\futures\process.py__init__Q   s    z_ThreadWakeup.__init__c             C   s   | j   | j  d S )N)r
   closer	   )r   r   r   r   r   T   s    
z_ThreadWakeup.closec             C   s   | j d d S )N    )r
   Z
send_bytes)r   r   r   r   wakeupX   s    z_ThreadWakeup.wakeupc             C   s   x| j  r| j   qW d S )N)r	   ZpollZ
recv_bytes)r   r   r   r   clear[   s    z_ThreadWakeup.clearN)__name__
__module____qualname__r   r   r   r   r   r   r   r   r   P   s   r   c              C   sH   da tt } x| D ]\}}|  qW x| D ]\}}|  q0W d S )NT)_global_shutdownlist_threads_wakeupsitemsr   join)r   _thread_wakeuptr   r   r   _python_exit`   s    r      c               @   s   e Zd Zdd Zdd ZdS )_RemoteTracebackc             C   s
   || _ d S )N)tb)r   r!   r   r   r   r   s   s    z_RemoteTraceback.__init__c             C   s   | j S )N)r!   )r   r   r   r   __str__u   s    z_RemoteTraceback.__str__N)r   r   r   r   r"   r   r   r   r   r    r   s   r    c               @   s   e Zd Zdd Zdd ZdS )_ExceptionWithTracebackc             C   s0   t t|||}d|}|| _d| | _d S )N z

"""
%s""")	tracebackformat_exceptiontyper   excr!   )r   r(   r!   r   r   r   r   y   s    
z _ExceptionWithTraceback.__init__c             C   s   t | j| jffS )N)_rebuild_excr(   r!   )r   r   r   r   
__reduce__~   s    z"_ExceptionWithTraceback.__reduce__N)r   r   r   r   r*   r   r   r   r   r#   x   s   r#   c             C   s   t || _| S )N)r    	__cause__)r(   r!   r   r   r   r)      s    
r)   c               @   s   e Zd Zdd ZdS )	_WorkItemc             C   s   || _ || _|| _|| _d S )N)futurefnargskwargs)r   r-   r.   r/   r0   r   r   r   r      s    z_WorkItem.__init__N)r   r   r   r   r   r   r   r   r,      s   r,   c               @   s   e Zd ZdddZdS )_ResultItemNc             C   s   || _ || _|| _d S )N)work_id	exceptionresult)r   r2   r3   r4   r   r   r   r      s    z_ResultItem.__init__)NN)r   r   r   r   r   r   r   r   r1      s   r1   c               @   s   e Zd Zdd ZdS )	_CallItemc             C   s   || _ || _|| _|| _d S )N)r2   r.   r/   r0   )r   r2   r.   r/   r0   r   r   r   r      s    z_CallItem.__init__N)r   r   r   r   r   r   r   r   r5      s   r5   c                   s*   e Zd Zd fdd	Z fddZ  ZS )
_SafeQueuer   c               s   || _ t j||d d S )N)ctx)pending_work_itemssuperr   )r   max_sizer7   r8   )	__class__r   r   r      s    z_SafeQueue.__init__c                sl   t |trZtt|||j}tdd||_	| j
|jd }|d k	rh|j| nt || d S )Nz

"""
{}"""r$   )
isinstancer5   r%   r&   r'   __traceback__r    formatr   r+   r8   popr2   r-   set_exceptionr9   _on_queue_feeder_error)r   eobjr!   	work_item)r;   r   r   rA      s    
z!_SafeQueue._on_queue_feeder_error)r   )r   r   r   r   rA   __classcell__r   r   )r;   r   r6      s   r6   c             g   s0   t | }x"tt|| }|s"d S |V  q
W d S )N)ziptuple	itertoolsislice)	chunksize	iterablesitchunkr   r   r   _get_chunks   s    rN   c                s    fdd|D S )Nc                s   g | ]} | qS r   r   ).0r/   )r.   r   r   
<listcomp>   s    z"_process_chunk.<locals>.<listcomp>r   )r.   rM   r   )r.   r   _process_chunk   s    	rQ   c          
   C   s^   y|  t|||d W n@ tk
rX } z"t||j}|  t||d W d d }~X Y nX d S )N)r4   r3   )r3   )putr1   BaseExceptionr#   r=   )result_queuer2   r4   r3   rB   r(   r   r   r   _sendback_result   s    
rU   c          
   C   s   |d k	r:y||  W n$ t k
r8   tjjddd d S X x| jdd}|d krb|t  d S y|j|j	|j
}W n> t k
r } z t||j}t||j|d W d d }~X Y nX t||j|d ~q<W d S )NzException in initializer:T)exc_info)block)r3   )r4   )rS   r   ZLOGGERZcriticalgetrR   osgetpidr.   r/   r0   r#   r=   rU   r2   )
call_queuerT   initializerinitargsZ	call_itemrrB   r(   r   r   r   _process_worker   s$    "r_   c             C   sx   xr|  rd S y|jdd}W n tjk
r4   d S X | | }|j rh|jt||j|j	|j
dd q| |= qqW d S )NF)rW   T)ZfullrX   queueZEmptyr-   Zset_running_or_notify_cancelrR   r5   r.   r/   r0   )r8   Zwork_idsr[   r2   rD   r   r   r   _add_call_item_to_queue   s     

ra   c          
      sF  d fdd} fdd}|j }	|j }
|	|
g}xt||  dd  D }t|| }d }d}|	|kry|	 }d}W q tk
r } ztt|||j	}W d d }~X Y qX n|
|krd}d }|
  |rt|  d k	rd	_d_d td
}|d k	r tdd| d|_x$| D ]\}}|j| ~q*W |
  x D ]}|  qXW |  d S t|tr|}|  s|  d S nL|d k	r||jd }|d k	r|jr|j|j n|j|j ~~|  | r:y$d k	rd_|s |  d S W n tk
r8   Y nX d q6W d S )Nc                  s   t p d kp jS )N)r   _shutdown_threadr   )executorr   r   shutting_down8  s    z/_queue_management_worker.<locals>.shutting_downc           	      s   t dd  D } | }d}xn||k r| dkrxBt|| D ]2}y d  |d7 }W q> tk
rn   P Y q>X q>W t dd  D } q W    x D ]}|  qW d S )Nc             s   s   | ]}|  V  qd S )N)is_alive)rO   pr   r   r   	<genexpr>>  s    zD_queue_management_worker.<locals>.shutdown_worker.<locals>.<genexpr>r   r   c             s   s   | ]}|  V  qd S )N)re   )rO   rf   r   r   r   rg   J  s    )sumvaluesrangeZ
put_nowaitr   r   r   )Zn_children_aliveZn_children_to_stopZn_sentinels_sentirf   )r[   	processesr   r   shutdown_worker<  s    
z1_queue_management_worker.<locals>.shutdown_workerc             S   s   g | ]
}|j qS r   )sentinel)rO   rf   r   r   r   rP   a  s    z,_queue_management_worker.<locals>.<listcomp>TFzKA child process terminated abruptly, the process pool is not usable anymorez^A process in the process pool was terminated abruptly while the future was running or pending.z
'''
r$   z''')r	   ra   ri   r   ZrecvrS   r%   r&   r'   r=   r   _brokenrb   BrokenProcessPoolr    r   r+   r   r-   r@   Z	terminater<   intr?   r2   r3   Z
set_resultr4   r   )Zexecutor_referencerl   r8   Zwork_ids_queuer[   rT   r   rd   rm   Zresult_readerZwakeup_readerZreadersZworker_sentinelsZreadycauseZ	is_brokenZresult_itemrB   Zbper2   rD   rf   r   )r[   rc   rl   r   _queue_management_worker  s    (




rs   c           	   C   sh   t rtrttda ytd} W n ttfk
r:   d S X | dkrHd S | dkrTd S d|  attd S )NTZSC_SEM_NSEMS_MAX   z@system provides too few semaphores (%d available, 256 necessary))_system_limits_checked_system_limitedNotImplementedErrorrY   ZsysconfAttributeError
ValueError)Z	nsems_maxr   r   r   _check_system_limits  s    r{   c             c   s.   x(| D ] }|   x|r$| V  qW qW d S )N)reverser?   )iterableZelementr   r   r   _chain_from_iterable_of_lists  s    
r~   c               @   s   e Zd ZdS )rp   N)r   r   r   r   r   r   r   rp     s   rp   c                   sh   e Zd ZdddZdd Zdd Zd	d
 Zejjj	e_	ddd fdd
Z
dddZejjj	e_	  ZS )ProcessPoolExecutorNr   c             C   s   t   |d krt pd| _n|dkr.td|| _|d krDt }|| _|d k	rbt|sbt	d|| _
|| _d | _i | _d| _t | _d| _d| _i | _| jt }t|| j| jd| _d| j_| | _t | _t | _d S )Nr   r   z"max_workers must be greater than 0zinitializer must be a callableF)r:   r7   r8   T) r{   rY   	cpu_count_max_workersrz   r   Zget_context_mp_contextcallable	TypeError_initializer	_initargs_queue_management_thread
_processesrb   	threadingZLock_shutdown_lockro   _queue_count_pending_work_itemsEXTRA_QUEUED_CALLSr6   _call_queueZ_ignore_epipeZSimpleQueue_result_queuer`   r   	_work_idsr   _queue_management_thread_wakeup)r   max_workersZ
mp_contextr\   r]   Z
queue_sizer   r   r   r     s8    



zProcessPoolExecutor.__init__c          	   C   sv   | j d krr| jfdd}|   tjtt| || j| j	| j
| j| j| jfdd| _ d| j _| j   | jt| j < d S )Nc             S   s   t jd |  d S )Nz?Executor collected: triggering callback for QueueManager wakeup)r   utildebugr   )r   r   r   r   r   
weakref_cb3  s    zFProcessPoolExecutor._start_queue_management_thread.<locals>.weakref_cbZQueueManagerThread)targetr/   nameT)r   r   _adjust_process_countr   ZThreadrs   weakrefrefr   r   r   r   r   Zdaemonstartr   )r   r   r   r   r   _start_queue_management_thread.  s     



z2ProcessPoolExecutor._start_queue_management_threadc             C   sT   xNt t| j| jD ]8}| jjt| j| j| j	| j
fd}|  || j|j< qW d S )N)r   r/   )rj   lenr   r   r   ZProcessr_   r   r   r   r   r   Zpid)r   r   rf   r   r   r   r   I  s    z)ProcessPoolExecutor._adjust_process_countc          	   O   s   | j  | jrt| j| jr&tdtr2tdt }t||||}|| j	| j
< | j| j
 |  j
d7  _
| j  |   |S Q R X d S )Nz*cannot schedule new futures after shutdownz6cannot schedule new futures after interpreter shutdownr   )r   ro   rp   rb   RuntimeErrorr   r   ZFuturer,   r   r   r   rR   r   r   r   )r   r.   r/   r0   fwr   r   r   submitT  s    

zProcessPoolExecutor.submitr   )timeoutrJ   c               s:   |dk rt dt jtt|t|d|i|d}t|S )Nr   zchunksize must be >= 1.rJ   )r   )rz   r9   mapr   rQ   rN   r~   )r   r.   r   rJ   rK   results)r;   r   r   r   k  s    zProcessPoolExecutor.mapTc          	   C   s   | j  d| _W d Q R X | jr6| j  |r6| j  d | _| jd k	rd| j  |r^| j  d | _d | _	d | _
| jr| j  d | _d S )NT)r   rb   r   r   r   r   r   r   Zjoin_threadr   r   )r   r   r   r   r   shutdown  s"    





zProcessPoolExecutor.shutdown)NNNr   )T)r   r   r   r   r   r   r   r   Executor__doc__r   r   rE   r   r   )r;   r   r     s    
C
r   )NN)1
__author__atexitrY   concurrent.futuresr   r`   r   Zmultiprocessingr   Zmultiprocessing.connectionr   Zmultiprocessing.queuesr   r   r   	functoolsr   rH   r%   WeakKeyDictionaryr   r   r   r   r   	Exceptionr    r#   r)   objectr,   r1   r5   r6   rN   rQ   rU   r_   ra   rs   rv   rw   r{   r~   ZBrokenExecutorrp   r   r   registerr   r   r   r   <module>.   sP   		

(& ! 9