/********************************************************************* */
/*  Program        : 0.Mispricingfactor.sas                                                  */
/*  Description    : Construct the Stambaugh-Yuan mispricing factor          */
/*                   (MISP), used in Table 1 of the paper.                                  */
/*                   At each month, NYSE, AMEX, and NASDAQ stocks with  */
/*                   prices of at least $5 are sorted independently by                  */
/*                   size (NYSE median) and by the mispricing score                */
/*                   (20th and 80th percentiles of the combined                           */
/*                   universe) into 2x3 portfolios.                                               */
/*                   MISP = Low - High: long the underpriced (low                    */
/*                   score) and short the overpriced (high score)                         */
/*                   portfolios.                                                                           */
/*                                                                                                              */
/*  Input : data\ Misp_Score.csv (mispricing score, downloaded from        */
/*          Robert Stambaugh's website)                                                        */
/*          data\ monthret_ff (from 0.DataDownload.sas)                               */
/*  Output: output\misp\ SAS datasets:                                                       */
/*          MISP_factor  (monthly mispricing factor)                                    */
/*          MISP_factory (annualized mispricing factor)                                */
/********************************************************************* */

%let projpath = D:\SAS\TFP\Dispersion_replication;
libname mydisp "&projpath.\data";
libname mymisp "&projpath.\output\misp";

/*1. Read the data*/
/*1-1. Mispricing score from Stambaugh's website*/
PROC IMPORT datafile="&projpath.\data\Misp_Score.csv"
 	OUT=mymisp.misp_score
 	DBMS=csv
 	REPLACE;
 	GETNAMES=Yes;
RUN;
proc sort data=mymisp.misp_score; by permno yyyymm; run;
data mymisp.misp_score2; set mymisp.misp_score;
Year = int(yyyymm/100);
Month = yyyymm-Year*100;
run;
/*1-2. CRSP monthly returns*/
data mymisp.crsp(keep=permno date prc ret exret ME lme siccd exchcd shrcd nyse50bp);
     set mydisp.monthret_ff;
     by permno;
     LME=lag(me);
     if first.permno then LME=me/(1+ret);
	 if (exchcd=1 or exchcd=2 or exchcd=3);
	 if (shrcd=10 or shrcd=11);
	 if prc>=5;
run;
proc sort data = mymisp.crsp;
    by date;
run;
proc univariate data=mymisp.crsp noprint;
    var lme;
	by date;
	output out=mymisp.tlmc sum=tlmc;
run;
data mymisp.crsp2; merge mymisp.tlmc mymisp.crsp;
    by date;
	wi = lme/tlmc;
run;

/*2. Construct portfolios sorted by size and by the mispricing score*/
/*****************************************************************************/
/*At each month, sort NYSE, AMEX, and NASDAQ stocks, excluding those with    */
/*prices less than $5, by size (NYSE median) and mispricing score            */
/*independently (2x3). Breakpoints for the mispricing score are the 20th and */
/*80th percentiles of the combined NYSE, AMEX, and NASDAQ universe.          */
/*****************************************************************************/
/*2-1.Match the mispricing score and the stock return at each month t*/
proc sql;
create table mymisp.PFpool
as select * from mymisp.misp_score2 as a, mymisp.crsp2 as b
where a.permno=b.permno
and a.year=year(b.date)
and a.month=month(b.date)
and (exchcd = 1 or exchcd = 2 or exchcd = 3)
and (shrcd = 10 or shrcd = 11)
and avg_score ne .
and exret ne .
order by date;
quit;
/*2-2.Breakpoints*/
proc univariate data=mymisp.PFpool noprint;
where exchcd=1 and shrcd in (10,11) and exchcd in (1,2,3);
var ME;
by date;
output out=mymisp.nyse_me  median = SIZEMEDN  ;
run;
proc univariate data=mymisp.PFpool noprint;
where shrcd in (10,11) and exchcd in (1,2,3);
var avg_score;
by date;
output out=mymisp.bp_misp  pctlpts=20 80 pctlpre=misp;
run;
proc sql;
create table mymisp.PFpool2 as
select a.*, b.sizemedn, c.misp20, c.misp80
from mymisp.PFpool as a, mymisp.nyse_me as b, mymisp.bp_misp as c
where a.date=b.date=c.date;
quit;

/*3.Create portfolios at the beginning of each month*/
/*Size portfolios: S[mall] or B[ig]                 */
/*Mispricing portfolios: H[igh], M[iddle], or L[ow] */
data mymisp.PF_sorting;
     set mymisp.PFpool2;
     if ME <= sizemedn                                        then SIZEport = 'S';
     else if ME > sizemedn                                    then SIZEport = 'B';
     if avg_score <= misp20                                   then MISPport = 'L';
     else if avg_score >  misp20 and avg_score <= misp80      then MISPport = 'M';
     else if avg_score  > misp80                              then MISPport = 'H';
if missing(sizeport)=0 and missing(MISPport)=0 then nonmissport=1;
else nonmissport=0;
keep permno date SIZEport MISPport nonmissport;
run;
proc sql;
create table mymisp.PF_return as
select a.date, a.permno, a.ret, a.exret, a.wi, b.date as portdate format date9., b.SIZEport, b.MISPport, b.nonmissport
from mymisp.crsp2 as a, mymisp.PF_sorting as b
where a.permno=b.permno
and intnx('month',a.date,0,'E')=
    intnx('month',b.date,1,'E')
order by date , SIZEport, MISPport;
quit;

/*4.Calculate the mispricing factor*/
proc univariate noprint data=mymisp.PF_return;
    var exret;
    by date SIZEport MISPport;
    weight wi;
    output out = mymisp.portret mean = vwret n = N;
run;
proc transpose data=mymisp.portret(keep=date SIZEport MISPport vwret) out=mymisp.portret2 (drop=_name_ _label_);
by date;
ID SIZEport MISPport;
Var vwret;
run;
data mymisp.MISP_factor(keep=date High Low MISP);
     set mymisp.portret2;
	 High=(BH+SH)/2;
	 Low=(BL+SL)/2;
	 MISP=Low - High;
run;
data mymisp.MISP_factor2;
     set mymisp.MISP_factor;
     if date > '01Jan1967'd;
	 if date < '01Jan2017'd;
run;
proc means data=mymisp.MISP_factor2;
run;

/*5.Annualize the mispricing factor*/
proc printto log = junk; run;
proc expand data = mymisp.portret2 out=mymisp.portret_y method=none;
    id date;
    convert bh = bhy  / transformin=(+1) transformout=(MOVPROD  12 -1 trimleft  11);
	convert bl = bly  / transformin=(+1) transformout=(MOVPROD  12 -1 trimleft  11);
	convert bm = bmy  / transformin=(+1) transformout=(MOVPROD  12 -1 trimleft  11);
	convert sh = shy  / transformin=(+1) transformout=(MOVPROD  12 -1 trimleft  11);
	convert sl = sly  / transformin=(+1) transformout=(MOVPROD  12 -1 trimleft  11);
	convert sm = smy  / transformin=(+1) transformout=(MOVPROD  12 -1 trimleft  11);
quit;
proc printto; run;
data mymisp.MISP_factory(keep=date High Low MISP);
     set mymisp.portret_y;
	 High=(BHy+SHy)/2;
	 Low=(BLy+SLy)/2;
	 MISP=Low - High;
run;
proc means data=mymisp.MISP_factory;
run;
