clear
%
rand('state',sum(100*clock))
%
% set calibration period and full run period (calibration + reconstruction)
earlylate=1;
startrunyear=850;
endrunyear=1980;
startcalyear=1900;
endcalyear=1980; 

% convert years to rows in the matrix
startrunyear=startrunyear-850+1;
endrunyear=1999-endrunyear; endrunyear=1150-endrunyear;

startcalyear=startcalyear-850+1;
endcalyear=1999-endcalyear; endcalyear=1150-endcalyear;
  ncut=startcalyear-startrunyear; %split for verification/calibration periods
  
  %%%%%%%%%%%%%%%%%%%%%%%%%%
  %%% Set parameters for pseudoproxy generation
  %%%%%%%%%%%%%%%%%%%%%%%%%%
  coeff=1.0; %mean proportion of signal in 'X'
  dcoeff=0.0; %range (std) of coeff around the mean
   np= 104; % number of proxies
  nproxies=np;
  inoise=1; % do you want noise? 1=yes, 0=no
  inl=0; % flag for non-local relationships (not implemented)
  a=1.0; % average fraction of dependence of proxy(t) on inst(t), lag proxy dependence
	%% b=1-a is average fraction of proxy(t) depending on inst(t-1)
  da=0.0;% range (std) about a
  rhon0=0; %lag-1 autocorellation of noise (red noise=pos, blue noise=neg)
  drhon=0.0; %range (std) about rhon0
  s1=0.4% ave signal:noise for 'X'
  r1=0.0; % range (std) about average s:n w.r.t 'X'
  pctwarm=0; % percent of warm season proxies
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% load model data
load converted.mat
load keepers.mat
% the second load is the columns in the matrix to be used

orig1=northtosouth(:,289:2304); % just keep 70n to 70s
orig1=orig1(:,[enough]);


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% set the lat/long values for the gridboxes

startlat=67.5;  % Northern boundary latitude (use negative if in southern hemisphere)
        endlat=-67.5;   % Southern boundary latitude (use positive if in northern hemisphere)
        gridstep=5;   % the gridbox resolution
        countlat=1;
        countlong=1;
        for i=startlat:-gridstep:endlat
        lat(countlat)=i;  % keep this for weighting the means later.
        for j=-177.5:gridstep:177.5
                longlat(countlong,1)=j;
                longlat(countlong,2)=i;
                countlong=countlong+1;
        end
        countlat=countlat+1;
        end
gridlocs=longlat;
gridlocs=longlat([enough],:);

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

  orig2=northtosouth(:,289:2304); 
  orig2=orig2(:,[enough]);% this was originally here for the warm season

  xloc1(:,1)=gridlocs(:,2);
  xloc1(:,2)=gridlocs(:,1);
  xloc2(:,1)=gridlocs(:,2);
  xloc2(:,2)=gridlocs(:,1);
orig1=orig1(startrunyear:endrunyear,:);
orig2=orig2(startrunyear:endrunyear,:);

% we only included annual proxies, but one could consider seasonal proxies as well
  pctannual=100-pctwarm
  nwarm=pctwarm*np/100; % number of warm season proxies
  nann = np-nwarm
%
%%% call the routine that makes the pseudoproxy networks
  [orig,xloc,xnorm, x, sd, proxy,proxynorm,ploc,merged,N,m1,m2,index1,index2]=pseudoproxytwo(orig1,orig2,xloc1,xloc2,inoise,s1,r1,coeff,dcoeff,inl,a,da,rhon0,drhon,np,nann,nwarm,ncut,earlylate);

clear merged % make sure there are no leftovers
% you could also load an existing proxy network here if you uncomment the next two lines.
%clear proxynorm
%load proxynorm

% reload the model data
load converted.mat
load keepers.mat
inst=northtosouth(:,289:2304); % just keep 70n to 70s
inst=inst(:,[enough]); % only reconstruct a subset of gridboxes based on the availability of actual instrumental data
inst(1:startcalyear-1,:)=NaN; % set verification to NaN
inst=inst(startrunyear:endrunyear,:); % cut at 1980
[nrows,ninst]=size(inst);

inst(startrunyear:startcalyear-1,:)=NaN; % remove the model data over the reconstruction period

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% calculate EOFs of model data

instmeans=nanmean(inst(startcalyear:endcalyear,:));
inst=inst-repmat(instmeans,nrows,1);
inststds=nanstd(inst(startcalyear:endcalyear,:));
inst=inst./repmat(inststds,nrows,1);
inst=inst(startcalyear:endcalyear,:);

[U,S,V] = svd(inst,0); % do the decomposition, U's are the PCs
instpcs=U(:,1:endcalyear-startcalyear+1);
insteofs=V; 
instsingvals=diag(S);

instevalues=diag(S.^2);
instpctvar=diag(S).^2./sum(diag(S).^2); % that gives the pct for each pc
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% now look at proxynetwork to figure out now many inst eofs to keep
% standardize the proxies over the calibration period
proxymeans=nanmean(proxynorm(startcalyear:endcalyear,:));
proxynorm=proxynorm-repmat(proxymeans,nrows,1);
proxystds=nanstd(proxynorm(startcalyear:endcalyear,:));
proxynorm=proxynorm./repmat(proxystds,nrows,1);
[U,S,V] = svd(proxynorm(startcalyear:endcalyear,:),0); % do the decomposition, U's are the PCs
proxysingvals=diag(S);
proxyevalues=diag(S.^2);
proxypctvar=diag(S).^2./sum(diag(S).^2); % that gives the pct for each pc
instpcstokeep=eigenselect(proxypctvar)% for "index only" set to 1, and load the index you want to reconstruct
keptpcs=instpcs(:,1:instpcstokeep);
kepteofs=insteofs(:,1:instpcstokeep);
kepteofs=kepteofs';
keptsingvals=instsingvals(1:instpcstokeep);
savepcs=keptpcs;
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
m1=instpcstokeep;ncols=m1;
ncols=ncols+np;


% standardize the retained model field PCs over the calibration period 
keptpcmeans=nanmean(keptpcs(startcalyear-startcalyear+1:endcalyear-startcalyear+1,:));
keptpcs=keptpcs-repmat(keptpcmeans,endcalyear-startcalyear+1,1);
keptpcstds=nanstd(keptpcs(startcalyear-startcalyear+1:endcalyear-startcalyear+1,:));
keptpcs=keptpcs./repmat(keptpcstds,endcalyear-startcalyear+1,1);


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%% combine the PCs to be reconstructed and the pseudoproxy network
augmented(startcalyear:endcalyear,1:m1)=keptpcs;
augmented(startrunyear:endrunyear,m1+1:ncols)= proxynorm;
%
merged(startrunyear:endrunyear,1:ncols)=NaN;
merged(ncut+1:N,1:m1)=keptpcs(:,1:m1);
merged(startrunyear:endrunyear,m1+1:ncols)=proxynorm;
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

ncut=startcalyear-startrunyear+1;


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% start the work of splitting the matrix into high-frequency and low-frequency bands
	cutoff=20; %years
	lowlim=1./(cutoff./2)
	[nrows,ncols]=size(merged) 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%  need to find where each series starts and stops. Series must be continuous
gotstart=zeros(1,ncols);
gotstop=zeros(1,ncols);
start=zeros(1,ncols);
stop=zeros(1,ncols);
stop(:)=nrows;
[lowa,lowb]=butter(5,lowlim,'low');


for i=1:nrows
	for j=1:ncols
		if ~isnan(merged(i,j)) & gotstart(j)~=1
			start(j)=i;
			gotstart(j)=1;
		elseif isnan(merged(i,j)) & gotstart(j)==1 & gotstop(j)~=1
			stop(j)=i-1;
			gotstop(j)=1;	
		end
	end
end
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%%%%	Apply the zero-phase butterworth filter 
	highf=zeros(nrows,ncols);
	lowf=zeros(nrows,ncols); 
	lowf=lowf./lowf;
	highf=highf./highf;
	for k=1:ncols
		clear y y2
		data=merged(start(k):stop(k),k);
		lowf(start(k):stop(k),k)=lowpassmin(data,1./cutoff);
		highf(start(k):stop(k),k)=data-lowf(start(k):stop(k),k);
    end
%
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%% Using the full data matrix, over the calibration period, figure out the ttls settings in regem
[U,S,V] = svd(lowf(startcalyear:endcalyear,ncols-np+1:ncols),0); % do the decomposition, U's are the PCs
lowfevalues=diag(S.^2);
lowfpctvar=diag(S).^2./sum(diag(S).^2); % that gives the pct for each pc
cumlowfpctvar=cumsum(lowfpctvar);

%%%% for the low-f, set ttls truncation to 50% of variance. It may be better to err on the side of picking one than 50%
%%%% depending on your application. You may want to comment out the 2nd line.
ttlslowfpar=length(find(cumlowfpctvar<0.5));  %
ttlslowfpar=ttlslowfpar+1;

[U,S,V] = svd(highf(startcalyear:endcalyear,ncols-np+1:ncols),0); % do the decomposition, economy style. U's are the PCs
highfevalues=diag(S.^2);
highfpctvar=diag(S).^2./sum(diag(S).^2); % that gives the pct for each pc
cumhighfpctvar=cumsum(highfpctvar);
ttlshighfpareigen=eigenselect(highfpctvar);

save ttlspicks ttlshighfpareigen ttlslowfpar

		save highfinput highf
		save lowfinput lowf

  save hybridsplitinput keptpcs kepteofs instpcstokeep keptsingvals keptpcmeans keptpcstds xloc index1 ncut merged augmented startrunyear endrunyear startcalyear endcalyear N m1 lat enough toofew nproxies x sd instmeans inststds


