B
    ërµdZí  ã               @   sj  d dl mZmZ d dlmZ d dlmZmZ ddlm	Z	m
Z
 ddlmZmZmZ d dlmZmZ d dlmZ d d	lZd d	lZd d
lmZ d dlmZmZmZmZ d dlm Z m!Z! d dl"m#Z# d dl$m%Z% d dl$m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ d d	l,Z,d dl-m.Z. d d	l/Z/d dl0m1Z1 d dl2m3Z3 d d	l4m5Z6 d d	l7m8Z9 d d	l:Z:d d	l;m<Z= d dl>m?Z? d dl>m@Z@ d dl>mAZA d dl>mBZB ddlCmDZD ddlEmFZF ddlGmHZHmIZI d dlJmKZK dd„ ZLdd„ ZMdd „ ZNd!d"„ ZOd#d$„ ZPd%d&„ ZQd'd(„ ZRd)d*„ ZSd+d,„ ZTd-d"„ ZOd.d/„ ZUd0d1„ ZVdHd2d3„ZWd4d5„ ZXd6d7„ ZYd8d9„ ZZd:d;„ Z[d<d=„ Z\d>d?„ Z]d@dA„ Z^dBdC„ Z_dDdE„ Z`dFdG„ Zad	S )Ié    )ÚrenderÚredirect)ÚHttpResponse)ÚQÚAvgé   )Ú
HUMINTFormÚ	OSINTForm)ÚOSINTÚHUMINTÚScoreUpdate)ÚauthenticateÚlogin)ÚreverseN)Útrain_test_split)ÚLogisticRegressionÚLinearRegressionÚSGDClassifierÚSGDRegressor)ÚGradientBoostingClassifierÚGradientBoostingRegressor)ÚKNeighborsClassifier)Úaccuracy_score)Úconfusion_matrixÚf1_scoreÚmean_squared_errorÚr2_scoreÚprecision_scoreÚrecall_score)ÚDocxTemplate)Úsettings)ÚMinMaxScaler)ÚPasswordResetView)ÚPasswordResetDoneView)ÚPasswordResetConfirmView)ÚPasswordResetCompleteView)Úcreate_gauge)Úcreate_pie_chart)Úupdate_scoreÚupdate_score_manual)ÚSMOTEc             C   s   d}t | dƒS )NÚredirect_toz
index.html)r   )ÚrequestZredirect_field_name© r-   ú!/var/www/html/tia/tiagen/views.pyÚindex#   s    r/   c             C   s  | j dkr| j d¡}|dkr8| j ¡ | _d| jd< n>|dkrX| j ¡ | _d| jd< n|dkrv| j ¡ | _d| jd< | j d	¡}|d
kr¢| j ¡ | _d
| jd< n|dkrÀ| j ¡ | _d| jd< t| jƒ}| ¡ rê| ¡  t| dd|iƒS ||jdœ}t| d|ƒS nt	ƒ }t| dd|iƒS )NÚPOSTÚ	OSINT_1_3Ú1ÚOSINT_1_3_1Ú2ÚOSINT_1_3_2Ú3ÚOSINT_1_3_3Ú	OSINT_2_4ÚbroadÚOSINT_2_4_1ÚspecificÚOSINT_2_4_2z
index.htmlÚform)r=   Úform_errorsz
osint.html)
Úmethodr0   ÚgetÚcopyr	   Úis_validÚsaver   Úerrorsr   )r,   Ú	osint_1_3Ú	osint_2_4r=   Úcontextr-   r-   r.   ÚaddOSINT'   s4    


rH   c       
      C   s–  | j dkrh| j d¡}|dkr8| j ¡ | _d| jd< nj|dkrX| j ¡ | _d| jd< nJ|dkrx| j ¡ | _d| jd< n*| j ¡ | _d	| jd< d	| jd< d	| jd< | j d
¡}|dkrÎ| j ¡ | _d| jd< nl|dkrî| j ¡ | _d| jd< nL|dkr| j ¡ | _d| jd< n*| j ¡ | _d	| jd< d	| jd< d	| jd< | j d¡}|dkrh| j ¡ | _d| jd< nn|dkrŠ| j ¡ | _d| jd< nL|dkr¬| j ¡ | _d| jd< n*| j ¡ | _d	| jd< d	| jd< d	| jd< | j d¡}|dkr| j ¡ | _d| jd< nn|dkr&| j ¡ | _d| jd< nL|dkrH| j ¡ | _d| jd< n*| j ¡ | _d	| jd< d	| jd< d	| jd< | j d¡}|dkr | j ¡ | _d| jd< nn|dkrÂ| j ¡ | _d| jd< nL|dkrä| j ¡ | _d| jd< n*| j ¡ | _d	| jd< d	| jd< d	| jd< tjj|d ¡ }t| jƒ}| 	¡ rL| 
¡  t| dd|iƒS |||jdœ}	t| d|	ƒS n*|}tjj|d ¡ }d|i}	t| d|	ƒS d S ) Nr0   Ú
HUMINT_0_1r2   ÚHUMINT_0_1_1r4   ÚHUMINT_0_1_2r6   ÚHUMINT_0_1_3Ú Ú
HUMINT_0_3ÚHUMINT_0_3_1ÚHUMINT_0_3_2ÚHUMINT_0_3_3Ú
HUMINT_0_4ÚHUMINT_0_4_1ÚHUMINT_0_4_2ÚHUMINT_0_4_3Ú
HUMINT_0_5ÚHUMINT_0_5_1ÚHUMINT_0_5_2ÚHUMINT_0_5_3Ú
HUMINT_0_6ÚHUMINT_0_6_1ÚHUMINT_0_6_2ÚHUMINT_0_6_3)Úidz
index.htmlr=   )Údefaultsr=   r>   zhumint.htmlr_   )r?   r0   r@   rA   r
   ÚobjectsÚfilterÚfirstr   rB   rC   r   rD   )
r,   r^   Ú
humint_0_1Ú
humint_0_3Ú
humint_0_4Ú
humint_0_5Ú
humint_0_6r_   r=   rG   r-   r-   r.   Ú	addHUMINTI   s¤    


























rh   c             C   s
   t | dƒS )Nzgeneratescore.html)r   )r,   r-   r-   r.   Úgeneratescoreª   s    ri   c             C   s–   | j dkrˆd| jkr^| j d¡}|r2| j d¡}nd}tj t|d¡ d¡}t| dd|iƒS d| jkr’| j d	d¡}|r’t	d
|dS n
t| dƒS d S )Nr0   Ú
searchformÚcanNamerM   )ÚcanName__icontainszsearchosint.htmlÚ	my_stringÚ
choiceformÚ
selectedidÚeditOSINTform)r^   )
r?   r0   r@   r
   r`   ra   r   Úorder_byr   r   )r,   rk   ÚqueryÚresultsro   r-   r-   r.   ÚsearchOSINT­   s    


rt   c             C   sš  | j dkrl| j d¡}|dkrL| j ¡ | _d| jd< d| jd< d| jd< nf|dkr€| j ¡ | _d| jd< d| jd< d| jd< n2|d	kr²| j ¡ | _d| jd< d| jd< d| jd< | j d
¡}|dkrè| j ¡ | _d| jd< d| jd< n*|dkr| j ¡ | _d| jd< d| jd< tjj|d ¡ }t| j|d}| 	¡ r>|j
d pJd |_|j
d p\d |_|j
d pnd |_|j
d p€d |_|j
d p’d |_|j
d p¤d |_|j
d p¶d |_|j
d pÈd |_|j
d pÚd |_|j
d pìd |_|j
d pþd |_|j
d pd |_|j
d p"d |_| ¡  t| dd|iƒS tjj|d ¡ }|||jd œ}t| d!|ƒS n*|}tjj|d ¡ }d"|i}t| d!|ƒS d S )#Nr0   r1   r2   r3   rM   r5   r7   r4   r6   r8   r9   r:   r<   r;   )r^   )ÚinstanceÚ	OSINT_1_1Ú	OSINT_1_2Ú	OSINT_1_4Ú	OSINT_1_5Ú	OSINT_2_1Ú	OSINT_2_2Ú	OSINT_2_3Ú	OSINT_2_5Ú	OSINT_3_1Ú	OSINT_3_3Ú	OSINT_3_5ÚcompanyNameÚ
companyLocz
index.htmlr=   )r_   r=   r>   zeditOSINTform.htmlr_   )r?   r0   r@   rA   r
   r`   ra   rb   r	   rB   Úcleaned_datarv   rw   rx   ry   rz   r{   r|   r}   r~   r   r€   r   r‚   rC   r   rD   )r,   r^   rE   rF   r_   r=   rG   r-   r-   r.   rp   ¿   sf    











rp   c             C   s¢   | j dkr”d| jkrj| j d¡}|r2| j d¡}nd}tj t|dt|dB ¡ dd¡}t| dd	|iƒS d
| jkrž| j dd¡}|ržt	d|dS n
t| dƒS d S )Nr0   rj   rk   rM   )rl   )ZsourceName__icontainsÚ
sourceNamezsearchhumint.htmlrm   rn   ro   ÚeditHUMINTform)r^   )
r?   r0   r@   r   r`   ra   r   rq   r   r   )r,   rk   rr   rs   ro   r-   r-   r.   ÚsearchHUMINTú   s    

$
r†   c             C   s–   | j dkrˆd| jkr^| j d¡}|r2| j d¡}nd}tj t|d¡ d¡}t| dd|iƒS d	| jkr’| j d
d¡}|r’t	d|dS n
t| dƒS d S )Nr0   rj   rk   rM   )rl   z-idzprefillhumint.htmlrm   rn   ro   rh   )r^   )
r?   r0   r@   r
   r`   ra   r   rq   r   r   )r,   rk   rr   rs   ro   r-   r-   r.   ÚprefillHUMINT  s    


r‡   c       
      C   sÌ  | j dkrž| j d¡}|dkr8| j ¡ | _d| jd< nj|dkrX| j ¡ | _d| jd< nJ|dkrx| j ¡ | _d| jd< n*| j ¡ | _d	| jd< d	| jd< d	| jd< | j d
¡}|dkrÎ| j ¡ | _d| jd< nl|dkrî| j ¡ | _d| jd< nL|dkr| j ¡ | _d| jd< n*| j ¡ | _d	| jd< d	| jd< d	| jd< | j d¡}|dkrh| j ¡ | _d| jd< nn|dkrŠ| j ¡ | _d| jd< nL|dkr¬| j ¡ | _d| jd< n*| j ¡ | _d	| jd< d	| jd< d	| jd< | j d¡}|dkr| j ¡ | _d| jd< nn|dkr&| j ¡ | _d| jd< nL|dkrH| j ¡ | _d| jd< n*| j ¡ | _d	| jd< d	| jd< d	| jd< | j d¡}|dkr | j ¡ | _d| jd< nn|dkrÂ| j ¡ | _d| jd< nL|dkrä| j ¡ | _d| jd< n*| j ¡ | _d	| jd< d	| jd< d	| jd< tjj|d ¡ }t| j|d}| 	¡ rp|j
d pFd |_|j
d pXd |_|j
d pjd |_|j
d p|d |_|j
d pŽd |_|j
d p d |_|j
d p²d |_|j
d pÄd |_|j
d pÖd |_|j
d pèd |_|j
d púd |_|j
d pd |_|j
d pd |_|j
d p0d |_|j
d pBd |_|j
d pTd |_| ¡  t| dd|iƒS tjj|d ¡ }|||jdœ}	t| d |	ƒS n*|}tjj|d ¡ }d!|i}	t| d |	ƒS d S )"Nr0   rI   r2   rJ   r4   rK   r6   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   )r^   )ru   Ú
HUMINT_0_2z
index.htmlr=   )r_   r=   r>   zeditHUMINTform.htmlr_   )r?   r0   r@   rA   r   r`   ra   rb   r   rB   rƒ   rJ   rK   rL   rˆ   rO   rP   rQ   rS   rT   rU   rW   rX   rY   r[   r\   r]   rC   r   rD   )
r,   r^   rc   rd   re   rf   rg   r_   r=   rG   r-   r-   r.   r…     sÆ    

























r…   c             C   s–   | j dkrˆd| jkr^| j d¡}|r2| j d¡}nd}tj t|d¡ d¡}t| dd|iƒS d| jkr’| j d	d¡}|r’t	d
|dS n
t| dƒS d S )Nr0   rj   rk   rM   )rl   zgeneratescore.htmlrm   rn   ro   ÚgeneratescoreML)r^   )
r?   r0   r@   r
   r`   ra   r   rq   r   r   )r,   rk   rr   rs   ro   r-   r-   r.   ri   ’  s    


c       -         s¦  ddl m} | jdkr¢tƒ }tjdddgd}xBtj ¡ D ]2}tj ¡ }t t	| 
¡ ƒ¡}tj ¡ }t t	| 
¡ ƒ¡}|j}	tjj|	djd	d
d ¡ }
tjj|
d}| tdƒ¡d }|d krÐt|dƒ}d}d}|d kræd}nt|d ||  ƒ| }|jd d …|j d¡|j d	¡|j d¡gf }|jdd	dddgdd}|jd d …|j d¡|j d	¡|j d¡|j d¡|j d¡gf }|jdd	dddgdd}|d  dd¡|d< |d  dd¡|d< | d¡}| d¡}tj| |j
¡|j|jd}tj| |j
¡|j|jd}| |¡}| |¡}||d	 |
k }||d	 |
k }|jdd	dddgdd}|jdd	dgdd}|jdd}t |¡ ¡ }tj|j d
d|j d
dgdd}t!d d!ƒ}t" #|¡}W d Q R X d}| $|¡}| %|¡}|dkr |d d …df }d}d"}t|||  ƒ| }|dkrV|d d …df }d"}d}t|||  ƒ| }d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2g‰ |j‡ fd3d4„|jD ƒd} t &d5¡}!|  ¡ }"|
g|"_|" 'd6¡ d7g|"_|"d7 j
}#|!d8 j
}$t(j)|#|$d9}%|%d: d }%t|%ƒ}%t!d;d!ƒ}t" #|¡}W d Q R X | $|¡}&|&d }&|&d }&|&d d: d }&t|&ƒ}&t!d<d!ƒ}t" #|¡}W d Q R X | $|¡}'|'d }'|'d d: d }'t|'dƒ}'|dkr¦d}(n|'| | d }(t|(dƒ}(t!d=d!ƒ}t" #|¡}W d Q R X | $|¡})t|)d ƒ})t*|)dƒ})t!d>d!ƒ}t" #|¡}W d Q R X | $|¡}*t|*d ƒ}*t*|*dƒ}*|*| d? }+t|+ƒ}+|j+|*|%|
|||&|+d@œd
dA}q>W |j,dBdCdD}|j-dCdE},t.| dF|,dGdHœƒS d S )INr   )ÚstatsÚGETzCandidate NamezB+B Predicted ScorezWeighted Average)Úcolumns)r^   rk   T)Úflat)rk   ÚlabelZ
label__avgr   i,  iR  zN/Aéd   r^   Úcommentr   r‚   )Úaxisr„   r:   r9   r<   r;   )rŒ   r/   )Údropzlogistic_regression_model.pklÚrbiô  rJ   rK   rL   rˆ   rO   rP   rQ   rS   rT   rU   rW   rX   rY   r[   r\   r]   c                s   g | ]}|ˆ kr|‘qS r-   r-   )Ú.0Úcol)Úitems_to_remover-   r.   ú
<listcomp>  s    z&scoreAllCandidates.<locals>.<listcomp>z	Todd2.csvz	Tests.csvÚValuesZWeight)Úweightsi&  zlinear_regression_model.pklzrf_regressor_model.pklzmlp_model.pklzsgd_model.pklé   )z	SGD ScorezWeighted AveragezCandidate NamezB+B Predicted ScorezLogistic RegressionzLinear RegressionzAvg LR + SGD)Úignore_indexz	SGD ScoreF)Ú	ascending)r/   zdataframeDisplay.htmlZ	allscores)Ú
html_tableÚdataframeType)/ZscipyrŠ   r?   r!   ÚpdÚ	DataFramer
   r`   ÚallÚlistÚvaluesr   r^   ra   Úvalues_listrb   Ú	aggregater   ÚroundÚintÚilocrŒ   Úget_locr’   ÚreplaceÚfillnaÚfit_transformr/   ÚjoinÚmeanÚ	transposeÚconcatÚreset_indexÚopenÚpickleÚloadÚpredictÚpredict_probaÚread_csvÚto_csvÚnpZaverageÚminÚappendÚsort_valuesÚto_htmlr   )-r,   rŠ   ÚscalerZ	allScoresÚentryÚOSINTquerysetÚOSINTdfÚHUMINTquerysetÚHUMINTdfr^   ÚcandidateNameÚentriesZaverage_labelÚmin_newÚmax_newZscaled_average_labelÚOSINTdf_saveÚHUMINTdf_saveÚfiltered_OSINTÚfiltered_HUMINTÚHUMINTnoTextÚOSINTnoTextÚmean_valuesÚavgHUMINTdfÚXÚfileÚmodelÚficoÚy_pred_probaÚ
badcanpredÚlogregscoreÚgoodcanpredZX_waÚdfÚtestsÚscoresr™   Zweighted_avgZlinearScoreZ	ranForestZdelta_percentageZmlpScoreÚsgdScoreZavgsgdlrr   r-   )r–   r.   ÚscoreAllCandidates¦  sÊ    


0D



"
















"rÜ   c          
   C   s´  | j dkr°| j d¡}t| j d¡ƒ}t|ƒ}|d }d}|d }d}|d }t|dƒ |d	krhd
}n^d|  kr|d	k r†n nd}n@d|  kršdk r¤n nd}n"d|  kr¸dk rÂn nd}nd}| j}	t||||	ƒ tdƒ}
||||d d |d d |d d |d d |d d |d d dœ	}|
 	|¡ |
 
dd¡ |
 
dd¡ |
 
dd¡ | d d¡}|d! }|d" }d#| }|
 |¡ ||||||||d d$œ}t	| d%d|iƒS d S )&Nr0   Úmy_variableZnumberInputÚnamerM   r   r‚   zscoreSpeedo.pngi   ÚExceptionaliä  z	Very Goodiž  ÚGoodiD  ÚFairÚPoorz%Candidate_Rating_Mockup_template.docxÚfactorsr   r   rš   é   é   é   )	ÚcanNameVariabler   r‚   ÚhFactorVar1ÚhFactorVar2ÚhFactorVar3ÚlFactorVar1ÚlFactorVar2ÚlFactorVar3z
speedo.pngzoverall.pngzpiechart_shared.pngzcanChar.pngzpiechart_unique.pngú z_report.docxz_report.pdfzpartnerReports/)r^   r   r‚   rÞ   ÚfullReportNameÚ
score_textÚscorerã   zscorereport.html)r?   r0   r@   r§   Úevalr&   Úuserr)   r   r   Úreplace_picrª   rC   )r,   r^   rÝ   ÚoverrideScoreZmyvarrÄ   r   r‚   rð   Úcurrent_userÚtemplateÚreplacementsÚcanNamenoSpacesÚfullReportNamedocÚfullReportNamepdfÚtemplateoutputrs   r-   r-   r.   ÚgeneratescoreOverride=  sT    







rý   c       7      C   sÚ  d}| j dkrÌtƒ }tjj|djddd ¡ }tjj|djddd ¡ }tjj|djddd ¡ }tj ¡ }t 	t
| ¡ ƒ¡}	tj ¡ }
t 	t
|
 ¡ ƒ¡}|	jd d …|	j d	¡|	j d¡|	j d
¡gf }|	jd	dd
ddgdd}	|jd d …|j d	¡|j d¡|j d¡|j d¡|j d
¡gf }|jd	dddd
gdd}|	d  dd¡|	d< |	d  dd¡|	d< | d¡}|	 d¡}	tj	| |	j¡|	j|	jd}	tj	| |j¡|j|jd}|	 |¡}	| |¡}|	|	d |k }||d |k }t|ƒ}t|ƒ}|jd	dddd
gdd}|jd	dd
gdd}|jdd}t 	|¡ ¡ }tj|jdd|jddgdd}tddƒ}t |¡}W d Q R X d}| |¡}| |¡}|dkrÔ|d d …df }d}d}t |||  ƒ| }|dkr
|d d …df }d}d}t |||  ƒ| }tddƒ}t |¡}W d Q R X | |¡} t!| d ƒ} t"| dƒ} t!| | d ƒ}!| j#}"t$|||!|"|ƒ t %d¡}#|#jd d …df  &¡ }$|#jd d …df  &¡ }%t 	|j'|jd |$|%dœ¡}&|& (d¡ d d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/g}'|&|&d0  )|'¡  }&|&|&d1  )|'¡  }&d2|&d3< d4|&jdd5…|&j d3¡f< d6|&jd5d7…|&j d3¡f< d8|&jd7d9…|&j d3¡f< d:|&jd9d;…|&j d3¡f< d<|&jd;d=…|&j d3¡f< d6|&jd=d>…|&j d3¡f< d:|&jd>d …|&j d3¡f< |&d3  *¡ }(t 	|(j|(jd?œ¡})|& +d3¡d@  ,¡  ¡ }*t-|*dAƒ |&|&d@ dk }+|+j.dBdC„ dd|+dD< |+|+d3 j/ 0d<¡  }+|+ +d3¡ .dEdC„ ¡jdd},|,j1dDdFdG},|,j(dHdFdI |&|&d@ dk }-|-j1d@dFdG}-|d k	r8t2|dJƒ |dKkrÌdL}.njdM|  krädKk rîn ndN}.nHdO|  krdMk rn ndP}.n&dQ|  kr(dOk r2n ndR}.ndS}.n„t2|!dJƒ |!dKkrRdL}.njdM|!  krjdKk rtn ndN}.nHdO|!  krŒdMk r–n ndP}.n&dQ|!  kr®dOk r¸n ndR}.ndS}.dT|krÒ| dTdU¡}|,jdV |,jdW |,jdX |-jdY |-jdZ |-jd[ g}/t3d\ƒ}0||||,jdV |,jdW |,jdX |-jdY |-jdZ |-jd[ d]œ	}1|0 4|1¡ |0 5d^dJ¡ |0 5d_d`¡ |0 5dadA¡ | dbd2¡}2|2dc }3|2dd }4de|3 }5|0 6|5¡ |||||3|.|!|/dfœ}6t4| dgdh|6iƒS t4| diƒS d S )jNzb0.9r‹   )r^   rk   T)r   r   r‚   r^   r   r   )r‘   r„   rŽ   r:   r9   r<   r;   r   )rŒ   r/   )r’   zlogistic_regression_model.pklr“   i,  iô  iR  zsgd_model.pklrš   zfeatures_inverses_5_30_23.csv)ÚCoefficientsz	Test DatazFeature NameszInverse FeatureszfullCoefs.csvrJ   rK   rL   rˆ   rO   rP   rQ   rS   rT   rU   rW   rX   rY   r[   r\   r]   zFeature NameszInverse FeaturesrM   ÚCategoryzInterpersonal Informationræ   zBusiness Development Acumené
   zLegal Skillsé   zMarket Presenceé   zOpen Source Intelligenceé   é"   )rÿ   ÚCountz	Test Datazpiechart_unique.pngc             S   s   | d | d  S )Nz	Test Datarþ   r-   )Úrowr-   r-   r.   Ú<lambda>ï  ó    z!generatescoreML.<locals>.<lambda>ÚProductc             S   s   |   dd¡S )Nrš   r	  )Únlargest)Úxr-   r-   r.   r  ó  r  F)Úbyrœ   zeachCat.csv)r/   zscoreSpeedo.pngi   rß   iä  z	Very Goodiž  rà   iD  rá   râ   ú&z&amp;)r   rš   )r   rš   )rš   rš   )éÿÿÿÿrä   )éþÿÿÿrä   )éýÿÿÿrä   z%Candidate_Rating_Mockup_template.docx)	rç   r   r‚   rè   ré   rê   rë   rì   rí   z
speedo.pngzoverall.pngzpiechart_shared.pngzcanChar.pngrî   z_report.docxz_report.pdfzpartnerReports/)r^   r   r‚   rÞ   rï   rð   rñ   rã   zscorereport.htmlrÝ   zgeneratescore.html)7r?   r!   r
   r`   ra   r¤   rb   r¡   rŸ   r    r¢   r£   r   r¨   rŒ   r©   r’   rª   r«   r¬   r/   r­   Úlenr®   r¯   r°   r±   r²   r³   r´   rµ   r¶   r§   r¦   rº   ró   r(   r·   ÚtolistZcoef_r¸   ÚisinZvalue_countsÚgroupbyÚsumr'   ÚapplyÚstrÚcontainsr¼   r&   r   r   rô   rC   )7r,   r^   rõ   Úversionr¾   rÄ   r   r‚   rÀ   rÁ   rÂ   rÃ   rÈ   rÉ   rÊ   rË   ZnumOSINTreportsZnumHUMINTreportsrÌ   rÍ   rÎ   rÏ   rÐ   rÑ   rÒ   rÓ   rÔ   rÕ   rÆ   rÇ   rÖ   r×   rÛ   rñ   rö   ZfeatureListZfeature_namesZinverse_featuresrØ   r–   Zcategory_countsZ	df_countsZ
grouped_dfZhighFactorsZeachCatZ
lowFactorsrð   Z
coef_tabler÷   rø   rù   rú   rû   rü   rs   r-   r-   r.   r‰   r  s   

0D



"














4

r‰   c          	   C   s\   t j tjd|¡}t j |¡rXt|dƒ*}t| ¡ dd}dt j 	|¡ |d< |S Q R X d S )NZpartnerReportsr“   zapplication/octet-stream)Úcontent_typezattachment; filename=zContent-Disposition)
ÚosÚpathr­   r    ÚBASE_DIRÚexistsr²   r   ÚreadÚbasename)r,   ÚfilenameÚ	file_pathÚfÚresponser-   r-   r.   Údownload_file@  s    r%  c          	   C   s  | j jr tƒ }tj ¡ }t| ¡ ƒ}t 	|¡}|j
ddddgdd}tj ¡ }t| ¡ ƒ}t 	|¡}|j
dgdd}tj||ddd	}|j
ddd
gdd}|d  dd¡|d< |d  dd¡|d< | d¡}||d dk }|jd d …|jdkf }	tj	| |	j¡|	j|	jd}	|d  dd„ ¡|d< |jd d …|jdkf }
t|	|
ddd\}}}}tddd}| ||¡\}}t||ddd\}}}}tƒ }| ||¡}tddƒ}t ||¡ W d Q R X | |¡}t||ƒ}d| }t||ƒ}d ||t dƒ¡}t!|ƒS t"| dddiƒS d S ) Nr   r^   r   r‚   r   )r‘   rk   Úouter)ÚonÚhowr„   r:   r9   r<   r;   r   rŽ   )rŒ   r/   c             S   s   | dkrdS dS )NéA   r   r   r-   )r  r-   r-   r.   r  o  r  ztrainlogreg.<locals>.<lambda>gš™™™™™É?é*   )Ú	test_sizeÚrandom_stater¡   )Zsampling_strategyr,  zlogistic_regression_model.pklÚwbzProgram executed successfully with an error rate of {:.2%}. Also f1 score of {:.2%}. Go to <a href='{}'>Score All Candidates</a>.rÜ   z
error.htmlÚmessagezAdmin access required.)#ró   Úis_superuserr!   r
   r`   r¡   r¢   r£   rŸ   r    r’   r   Úmergerª   r«   r¨   rŒ   r¬   r/   r  r   r*   Úfit_resampler   Úfitr²   r³   Údumprµ   r   r   Úformatr   r   r   )r,   r¾   rÀ   ÚOSINTvaluesÚtrainosintdfrÂ   ÚHUMINTvaluesÚtrainhumintdfÚtrainHUMOSINTdfrÐ   ÚyÚX_trainÚX_testÚy_trainÚy_testÚsmoteÚX_train_resampledÚy_train_resampledÚlogregrÒ   rÑ   Úy_predÚaccuracyÚ
error_rateÚf1r.  r-   r-   r.   ÚtrainlogregH  sF    








rG  c          	   C   sÆ  t ƒ }tj ¡ }t| ¡ ƒ}t |¡}|jddddgdd}t	j ¡ }t| ¡ ƒ}t |¡}|jdgdd}tj
||ddd	}|jddd
gdd}|d  dd¡|d< |d  dd¡|d< | d¡}||d dk }|jd d …|jdkf }	tj| |	j¡|	j|	jd}	|d  dd„ ¡|d< |jd d …|jdkf }
t|	|
ddd\}}}}tdd}| ||¡}| ||¡}tddƒ}t ||¡ W d Q R X | |¡}t||ƒ}d| }d |tdƒ¡}t|ƒS )Nr   r^   r   r‚   r   )r‘   rk   r&  )r'  r(  r„   r:   r9   r<   r;   r   rŽ   )rŒ   r/   c             S   s   t | d d d ƒS )Nr   i&  i,  )r§   )r  r-   r-   r.   r  ®  r  ztrainknn.<locals>.<lambda>gš™™™™™É?r*  )r+  r,  r   )Zn_neighborszknn_model.pklr-  zYProgram executed successfully with an error rate of {:.2%}. Go to <a href='{}'>index</a>.r/   )r!   r
   r`   r¡   r¢   r£   rŸ   r    r’   r   r0  rª   r«   r¨   rŒ   r¬   r/   r  r   r   r2  rñ   r²   r³   r3  rµ   r   r4  r   r   )r,   r¾   rÀ   r5  r6  rÂ   r7  r8  r9  rÐ   r:  r;  r<  r=  r>  ZneighrÒ   rñ   rÑ   rC  rD  rE  r.  r-   r-   r.   Útrainknn  s<    







rH  c          	   C   s¤  t ƒ }tj ¡ }t| ¡ ƒ}t |¡}|jddddgdd}t	j ¡ }t| ¡ ƒ}t |¡}|jdgdd}tj
||ddd	}|jddd
gdd}|d  dd¡|d< |d  dd¡|d< | d¡}||d dk }|jd d …|jdkf }	tj| |	j¡|	j|	jd}	|jd d …|jdkf }
t|	|
ddd\}}}}tƒ }| ||¡}tddƒ}t ||¡ W d Q R X | |¡}t||ƒ}t||ƒ}d ||tdƒ¡}t|ƒS )Nr   r^   r   r‚   r   )r‘   rk   r&  )r'  r(  r„   r:   r9   r<   r;   r   rŽ   )rŒ   r/   gš™™™™™É?r*  )r+  r,  zlinear_regression_model.pklr-  z~Program executed successfully with an mse rate of {:.2}. Also r2 score of {:.2%}. Go to <a href='{}'>Score All Candidates</a>.rÜ   )r!   r
   r`   r¡   r¢   r£   rŸ   r    r’   r   r0  rª   r«   r¨   rŒ   r¬   r/   r   r   r2  r²   r³   r3  rµ   r   r   r4  r   r   )r,   r¾   rÀ   r5  r6  rÂ   r7  r8  r9  rÐ   r:  r;  r<  r=  r>  ZlinregrÒ   rÑ   rC  ÚmseZr2r.  r-   r-   r.   ÚtrainlinregÁ  s8    







rJ  c          	   C   sÄ  ddl m} ddlm} ddlm} tƒ }tj 	¡ }t
| ¡ ƒ}t |¡}|jddddgd	d
}tj 	¡ }t
| ¡ ƒ}	t |	¡}
|
jdgd	d
}
tj|
|ddd}|jdddgd	d
}|d  dd	¡|d< |d  dd	¡|d< | d¡}||d dk }|jd d …|jdkf }tj| |j¡|j|jd}|jd d …|jdkf }|||ddd\}}}}|ddd}| ||¡}tddƒ}t ||¡ W d Q R X | |¡}| ||¡}d |tdƒ¡}t|ƒS )Nr   )ÚRandomForestRegressor)r   )r   r   r^   r   r‚   r   )r‘   rk   r&  )r'  r(  r„   r:   r9   r<   r;   rŽ   )rŒ   r/   gš™™™™™É?r*  )r+  r,  i  )Zn_estimatorsr,  zrf_regressor_model.pklr-  zjProgram executed successfully with an accuracy rate of {:.2}. Go to <a href='{}'>Score All Candidates</a>.rÜ   ) Úsklearn.ensemblerK  Úsklearn.model_selectionr   Úsklearn.metricsr   r!   r
   r`   r¡   r¢   r£   rŸ   r    r’   r   r0  rª   r«   r¨   rŒ   r¬   r/   r2  r²   r³   r3  rµ   rñ   r4  r   r   )r,   rK  r   r   r¾   rÀ   r5  r6  rÂ   r7  r8  r9  rÐ   r:  r;  r<  r=  r>  Zrf_regressorrÒ   rÑ   rC  Úaccr.  r-   r-   r.   Útrainrandomforestô  s<    





rP  c                s$  ddl m} tddƒ}tjdddgd}xà|D ]Ö‰ tƒ }tj ¡ }t	| 
¡ ƒ}t |¡}|jd	d
ddgdd}tj ¡ }t	| 
¡ ƒ}	t |	¡}
|
jd	gdd}
tj|
|ddd}|jdd
dgdd}|d  dd¡|d< |d  dd¡|d< | d¡}||d dk }|d  ‡ fdd„¡|d< |jd d …|jdkf }tj| |j
¡|j|jd}|jd d …|jdkf }t||ddd\}}}}tdd}| ||¡\}}t||ddd\}}}}tƒ }| ||¡}| |¡}t||ƒ}d| }|||ƒ}|jˆ ||dœd d!}q0W | ¡ }t| d"d#|iƒS )$Nr   )r   é   éP   zthreshold valzerror rate valz	f1 scores)rŒ   r   r^   r   r‚   r   )r‘   rk   r&  )r'  r(  r„   r:   r9   r<   r;   rŽ   c                s   | ˆ krdS dS )Nr   r   r-   )r  )Ú	thresholdr-   r.   r  L  r  z$determineThreshold.<locals>.<lambda>)rŒ   r/   gš™™™™™É?r*  )r+  r,  )r,  )zthreshold valzerror rate valz	f1 scoresT)r›   zdataframeDisplay.htmlr   )rN  r   ÚrangerŸ   r    r!   r
   r`   r¡   r¢   r£   r’   r   r0  rª   r«   r  r¨   rŒ   r¬   r/   r   r*   r1  r   r2  rµ   r   r»   r½   r   )r,   r   Z
thresholdsZthresholdMatrixr¾   rÀ   r5  r6  rÂ   r7  r8  r9  rÐ   r:  r;  r<  r=  r>  r?  r@  rA  rB  rÒ   rC  rD  rE  rF  r   r-   )rS  r.   ÚdetermineThreshold.  sH    









rU  c          	   C   sÜ  | j jrÈtƒ }tj ¡ }t| ¡ ƒ}t 	|¡}|j
ddddgdd}tj ¡ }t| ¡ ƒ}t 	|¡}|j
dgdd}tj||ddd	}|j
ddd
gdd}|d  dd¡|d< |d  dd¡|d< | d¡}||d dk }|jd d …|jdkf }	tj	| |	j¡|	j|	jd}	|d  dd„ ¡|d< | d¡ |jd d …|jdkf }
t|	|
ddd\}}}}tddd}| ||¡}tddƒ}t ||¡ W d Q R X | |¡}t||ƒ}d |tdƒ¡}t|ƒS t | ddd iƒS d S )!Nr   r^   r   r‚   r   )r‘   rk   r&  )r'  r(  r„   r:   r9   r<   r;   r   rŽ   )rŒ   r/   c             S   s   t | d d d ƒS )Nr   i&  i,  )r§   )r  r-   r-   r.   r  —  r  ztrainsgd.<locals>.<lambda>ztrainingdatasetV1.csvgš™™™™™É?r*  )r+  r,  iè  )Zmax_iterr,  zsgd_model.pklr-  zbSGD executed successfully with an mse rate of {:.2f}. Go to <a href='{}'>Score All Candidates</a>.rÜ   z
error.htmlr.  zAdmin access required.)!ró   r/  r!   r
   r`   r¡   r¢   r£   rŸ   r    r’   r   r0  rª   r«   r¨   rŒ   r¬   r/   r  r¸   r   r   r2  r²   r³   r3  rµ   r   r4  r   r   r   )r,   r¾   rÀ   r5  r6  rÂ   r7  r8  r9  rÐ   r:  r;  r<  r=  r>  Zsgd_regressorrÒ   rÑ   rC  rI  r.  r-   r-   r.   Útrainsgdi  s>    








rV  c          	   C   s  dd l }tƒ }tj ¡ }t| ¡ ƒ}t |¡}|j	ddddgdd}t
j ¡ }t| ¡ ƒ}t |¡}|j	dgdd}tj||dd	d
}	|	j	dddgdd}	|	d  dd¡|	d< |	d  dd¡|	d< |	 d¡}	|	|	d dk }	|	jd d …|	jdkf }
tj| |
j¡|
j|
jd}
|	d d |	d< |	d  dd„ ¡|	d< |	jd d …|	jdkf }t|
|ddd\}}}}|j |jjjdddg¡}|jjjdd}|j|dd |j||ddd}td d!ƒ}t ||¡ W d Q R X | |¡}|j ||dd"}d# !|t"d$ƒ¡}t#|ƒS )%Nr   r   r^   r   r‚   r   )r‘   rk   r&  )r'  r(  r„   r:   r9   r<   r;   rŽ   )rŒ   r/   gÍÌÌÌÌÌð?c             S   s   t | d d d ƒS )Nr   i&  i,  )r§   )r  r-   r-   r.   r  Ú  r  ztrainsgdTF.<locals>.<lambda>gš™™™™™É?r*  )r+  r,  )é7   )Zinput_shapegü©ñÒMbP?)Zlearning_rater   )Ú	optimizerZlossr   )ZepochsÚverbosezsgdTF_model.pklr-  )rY  zfProgram executed successfully with an mse rate of {:.2%}. Go to <a href='{}'>Score All Candidates</a>.rÜ   )$Z
tensorflowr!   r
   r`   r¡   r¢   r£   rŸ   r    r’   r   r0  rª   r«   r¨   rŒ   r¬   r/   r  r   ZkerasZ
SequentialZlayersZDenseZ
optimizersZAdamÚcompiler2  r²   r³   r3  rµ   Zevaluater4  r   r   )r,   Útfr¾   rÀ   r5  r6  rÂ   r7  r8  r9  rÐ   r:  r;  r<  r=  r>  ZsgdrX  rÒ   rÑ   rC  rI  r.  r-   r-   r.   Ú
trainsgdTF¯  sB    





r\  c             C   sh  | j dkrZd| jkr`| j d¡}|r4| j d¡}nd}tj t|d¡ d¡}t| dd|iƒS d| jkrd| j d	d¡}|rdt	j t|d¡ d
¡}t
 t| ¡ ƒ¡}|jd }|jddgdd}d|j|d dkdf< d|j|d dkdf< |jddid}|jddid}|jddid}|d
 j d¡|d
< |jdd}t| d|d||dœƒS n
t| dƒS d S )Nr0   rj   rk   rM   )rl   zlookupscores.htmlrm   rn   ro   Ú	timestamp)r   ÚcanIDr^  r   )r‘   ÚmanualoverrideFzN/ATÚ	algorithmzManual Override?)rŒ   r^   zScore IDÚVersionz%m-%d-%y %H:%M)r/   zdataframeDisplay.htmlZ
scoreTable)r   rž   rk   r^  )r?   r0   r@   r
   r`   ra   r   rq   r   r   rŸ   r    r¢   r£   Úlocr’   ÚrenameÚdtÚstrftimer½   )r,   rk   rr   rs   rÚ   Zscoresdfr^  r   r-   r-   r.   Úlookupscoresø  s0    

rf  c          	   C   sR   | j jr>tddƒ}| ¡ }W d Q R X | ¡  t| dd|iƒS t| dddiƒS d S )Nzlogfile.logÚrz	logs.htmlÚlogsz
error.htmlr.  zAdmin access required.)ró   r/  r²   Ú	readlinesr   r   )r,   Zlog_filerh  r-   r-   r.   Ú	view_logs  s    rj  )N)bZdjango.shortcutsr   r   Údjango.httpr   Údjango.db.modelsr   r   Úformsr   r	   Úmodelsr
   r   r   Údjango.contrib.authr   r   Údjango.urlsr   ZpandasrŸ   Znumpyr¹   rM  r   Zsklearn.linear_modelr   r   r   r   rL  r   r   Zsklearn.neighborsr   rN  r   r   r   r   r   r   r   r³   Zdocxtplr   r  Údjango.confr    Zsklearn.preprocessingr!   Zplotly.graph_objsZ
graph_objsZgoZ	plotly.ioÚioZpioÚ
subprocessZmatplotlib.pyplotZpyplotZpltÚdjango.contrib.auth.viewsr"   r#   r$   r%   Zspeedor&   Z	pie_chartr'   ZupdateScorer(   r)   Zimblearn.over_samplingr*   r/   rH   rh   ri   rt   rp   r†   r‡   r…   rÜ   rý   r‰   r%  rG  rH  rJ  rP  rU  rV  r\  rf  rj  r-   r-   r-   r.   Ú<module>   sp    "a;s 5
 OE43:;FI