Modified files for strain-specific residual variance option

src/gemma.cpp

@@ -337,6 +337,16 @@ void GEMMA::PrintHelp(size_t option) {
     cout << "                  variable for individual 2" << endl;
     cout << "                  ..." << endl;
     cout << "          missing value: NA" << endl;
+    cout << " -resid   [filename]     "
+         << " residual variance file contains a diagonal matrix of positive values to be used "
+         << "directly as for the residual variance---each diagonal element corresponds to an "
+         << "individual, in which each value is the empirical residual variance (sigmasq) based"
+         << "on a trait value calculated from multiple replicates for this individual" 
+         << endl;
+    cout << "          format: variable for individual 1" << endl;
+    cout << "                  variable for individual 2" << endl;
+    cout << "                  ..." << endl;
+    cout << "          missing value: NA" << endl;
     cout << " -k        [filename]     "
          << " specify input kinship/relatedness matrix file name" << endl;
     cout << " -mk       [filename]     "
@@ -824,6 +834,14 @@ void GEMMA::Assign(int argc, char **argv, PARAM &cPar) {
       str.clear();
       str.assign(argv[i]);
       cPar.file_weight = str;
+    } else if (strcmp(argv[i], "-resid") == 0) {
+      if (argv[i + 1] == NULL || argv[i + 1][0] == '-') {
+        continue;
+      }
+      ++i;
+      str.clear();
+      str.assign(argv[i]);
+      cPar.file_resid = str;
     } else if (strcmp(argv[i], "-wsnp") == 0) {
       if (argv[i + 1] == NULL || argv[i + 1][0] == '-') {
         continue;
@@ -1732,6 +1750,7 @@ void GEMMA::BatchRun(PARAM &cPar) {
     gsl_matrix *Y = gsl_matrix_safe_alloc(cPar.ni_test, cPar.n_ph);
     gsl_matrix *W = gsl_matrix_safe_alloc(Y->size1, cPar.n_cvt);
     gsl_matrix *G = gsl_matrix_safe_alloc(Y->size1, Y->size1);
+    gsl_matrix *sigmasq = gsl_matrix_safe_alloc(cPar.n_resid, cPar.n_resid);
     gsl_matrix *U = gsl_matrix_safe_alloc(Y->size1, Y->size1);
     gsl_matrix *UtW = gsl_matrix_safe_alloc(Y->size1, W->size2);
     gsl_matrix *UtY = gsl_matrix_safe_alloc(Y->size1, Y->size2);
@@ -1821,7 +1840,8 @@ void GEMMA::BatchRun(PARAM &cPar) {
       gsl_matrix *se_B = gsl_matrix_safe_alloc(cPar.n_ph, W->size2);
 
       // obtain estimates
-      CalcMvLmmVgVeBeta(eval, UtW, UtY, cPar.em_iter, cPar.nr_iter,
+      MVLMM cMvlmm;
+      cMvlmm.CalcMvLmmVgVeBeta(eval, U, sigmasq, UtW, UtY, cPar.em_iter, cPar.nr_iter,
                         cPar.em_prec, cPar.nr_prec, cPar.l_min, cPar.l_max,
                         cPar.n_region, Vg, Ve, B, se_B);
 
@@ -2564,6 +2584,7 @@ void GEMMA::BatchRun(PARAM &cPar) {
     gsl_matrix *UtW = gsl_matrix_calloc(Y->size1, W->size2);
     gsl_matrix *UtY = gsl_matrix_calloc(Y->size1, Y->size2);
     gsl_vector *eval = gsl_vector_calloc(Y->size1);
+    gsl_matrix *sigmasq = gsl_matrix_calloc(cPar.n_resid, cPar.n_resid);
     gsl_vector *env = gsl_vector_safe_alloc(Y->size1);
     gsl_vector *weight = gsl_vector_safe_alloc(Y->size1);
     debug_msg("Started on LMM");
@@ -2611,6 +2632,8 @@ void GEMMA::BatchRun(PARAM &cPar) {
           }
         }
       }
+      // Need some way of reading the residual variance file and then
+      // assigning it to what would have been the ..+ 1 part of the computation
 
       // eigen-decomposition and calculate trace_G - main track
       cout << "Start Eigen-Decomposition..." << endl;
@@ -2827,15 +2850,15 @@ void GEMMA::BatchRun(PARAM &cPar) {
 
           if (!cPar.file_bfile.empty()) {
             if (cPar.file_gxe.empty()) {
-              cMvlmm.AnalyzePlink(U, eval, UtW, UtY);
+              cMvlmm.AnalyzePlink(U, eval, sigmasq, UtW, UtY);
             } else {
-              cMvlmm.AnalyzePlinkGXE(U, eval, UtW, UtY, env);
+              cMvlmm.AnalyzePlinkGXE(U, eval, sigmasq, UtW, UtY, env);
             }
           } else {
             if (cPar.file_gxe.empty()) {
-              cMvlmm.AnalyzeBimbam(U, eval, UtW, UtY);
+              cMvlmm.AnalyzeBimbam(U, eval, sigmasq, UtW, UtY);
             } else {
-              cMvlmm.AnalyzeBimbamGXE(U, eval, UtW, UtY, env);
+              cMvlmm.AnalyzeBimbamGXE(U, eval, sigmasq, UtW, UtY, env);
             }
           }
 

src/gemma_io.cpp

@@ -34,6 +34,7 @@
 #include <stdlib.h>
 #include <string>
 #include <vector>
+#include <limits>
 
 #include "gsl/gsl_blas.h"
 #include "gsl/gsl_cdf.h"
@@ -150,7 +151,7 @@ std::istream &safeGetline(std::istream &is, std::string &t) {
 
 // Read SNP file. A single column of SNP names.
 bool ReadFile_snps(const string file_snps, set<string> &setSnps) {
-  debug_msg("enter ReadFile_snps");
+  debug_msg("entered");
   setSnps.clear();
 
   igzstream infile(file_snps.c_str(), igzstream::in);
@@ -329,9 +330,6 @@ bool ReadFile_anno(const string &file_anno, map<string, string> &mapRS2chr,
     mapRS2bp[rs] = b_pos;
     mapRS2cM[rs] = cM;
   }
-  // for (auto& [key, value] : mapRS2bp) {
-  //     cerr << key << endl;
-  //}
 
   infile.close();
   infile.clear();
@@ -509,6 +507,84 @@ bool ReadFile_cvt(const string &file_cvt, vector<int> &indicator_cvt,
   return true;
 }
 
+bool ReadFile_resid(const std::string &file_resid, std::vector<int> &indicator_resid,
+                    gsl_matrix *resid, size_t &n_resid) {
+    debug_msg("entered");
+    indicator_resid.clear();
+
+    std::ifstream infile(file_resid.c_str(), std::ifstream::in);
+    if (!infile) {
+        std::cout << "error! fail to open residual variance file: " << file_resid << std::endl;
+        return false;
+    }
+
+    std::string line;
+    char *ch_ptr;
+    double d;
+    int flag_na = 0;
+    size_t row = 0;
+
+    // Read and parse each line
+    while (!safeGetline(infile, line).eof()) {
+        size_t col = 0;
+        flag_na = 0;
+
+        ch_ptr = strtok((char *)line.c_str(), " ,\t");
+        while (ch_ptr != NULL) {
+            if (strcmp(ch_ptr, "NA") == 0) {
+                flag_na = 1;
+                d = -9; // Handle missing value
+            } else {
+                d = atof(ch_ptr);
+            }
+
+            // Set value in gsl_matrix
+            gsl_matrix_set(resid, row, col, d);
+            col++;
+
+            ch_ptr = strtok(NULL, " ,\t");
+        }
+
+        if (flag_na == 0) {
+            indicator_resid.push_back(1); // No missing values
+        } else {
+            indicator_resid.push_back(0); // Contains missing values
+        }
+
+        if (row == 0) {
+            n_resid = col; // Set the number of residuals per row
+        } else if (n_resid != col) {
+            std::cout << "error! number of residuals in row " << row
+                      << " does not match other rows." << std::endl;
+            return false;
+        }
+
+        row++;
+    }
+
+    if (indicator_resid.empty()) {
+        n_resid = 0;
+    }
+
+    // Debugging step: Print the residual variance matrix
+    std::cout << "DEBUG: n_resid = " << n_resid << std::endl;
+    if (resid != nullptr) {
+        for (size_t i = 0; i < n_resid; ++i) {
+            for (size_t j = 0; j < n_resid; ++j) {
+                std::cout << gsl_matrix_get(resid, i, j) << " ";
+            }
+            std::cout << std::endl;
+        }
+    } else {
+        std::cout << "ERROR: Residual variance matrix not initialized." << std::endl;
+    }
+
+    infile.close();
+    infile.clear();
+
+    return true;
+}
+
 // Read .bim file.
 bool ReadFile_bim(const string &file_bim, vector<SNPINFO> &snpInfo) {
   debug_msg("entered");
@@ -684,9 +760,10 @@ bool ReadFile_geno(const string &file_geno, const set<string> &setSnps,
   double maf, geno, geno_old;
   size_t n_miss;
   size_t n_0, n_1, n_2;
+
   double min_g = std::numeric_limits<float>::max();
   double max_g = std::numeric_limits<float>::min();
-
+  
   int flag_poly;
 
   int ni_total = indicator_idv.size();
@@ -699,9 +776,6 @@ bool ReadFile_geno(const string &file_geno, const set<string> &setSnps,
   file_pos = 0;
   auto count_warnings = 0;
   auto infilen = file_geno.c_str();
-  // for (auto& [key, value] : mapRS2bp) {
-  //      cerr << key << endl;
-  // }
   while (!safe_get_line(infile, line).eof()) {
     ch_ptr = strtok_safe2((char *)line.c_str(), " ,\t",infilen);
     rs = ch_ptr;
@@ -1412,6 +1486,57 @@ void ReadFile_eigenD(const string &file_kd, bool &error, gsl_vector *eval) {
   return;
 }
 
+//This is the second mention of this function...not sure why it's here or if we need it?
+//void ReadFile_resid(const string &file_resid, bool &error, gsl_matrix *resid) {
+//  debug_msg("entered");
+//  igzstream infile(file_resid.c_str(), igzstream::in);
+//  if (!infile) {
+//    cout << "error! fail to open the residual variance file: " << file_resid << endl;
+//    error = true;
+//    return;
+//  }
+
+//  size_t n_row = resid->size1, n_col = resid->size2, i_row = 0, i_col = 0;
+
+//  gsl_matrix_set_zero(resid); //change this so that resid = V_e somehow
+
+//  string line;
+//  char *ch_ptr;
+//  double d;
+
+//  while (getline(infile, line)) {
+//    if (i_row == n_row) {
+//      cout << "error! number of rows in the residual variance file is larger "
+//           << "than expected." << endl;
+//      error = true;
+//    }
+
+//    i_col = 0;
+//    ch_ptr = strtok((char *)line.c_str(), " ,\t");
+//    while (ch_ptr != NULL) {
+//      if (i_col == n_col) {
+//        cout << "error! number of columns in the residual variance file "
+//             << "is larger than expected, for row = " << i_row << endl;
+//        error = true;
+//      }
+
+//      d = atof(ch_ptr);
+//      gsl_matrix_set(resid, i_row, i_col, d);
+//      i_col++;
+
+//      ch_ptr = strtok(NULL, " ,\t");
+//    }
+
+//    i_row++;
+//  }
+
+//  infile.close();
+//  infile.clear();
+
+//  return;
+//}
+
+
 // Read bimbam mean genotype file and calculate kinship matrix.
 bool BimbamKin(const string file_geno, const set<string> ksnps,
                vector<int> &indicator_snp, const int k_mode,

src/gemma_io.h

@@ -86,6 +86,8 @@ void ReadFile_mk(const string &file_mk, vector<int> &indicator_idv,
                  gsl_matrix *G);
 void ReadFile_eigenU(const string &file_u, bool &error, gsl_matrix *U);
 void ReadFile_eigenD(const string &file_d, bool &error, gsl_vector *eval);
+bool ReadFile_resid(const string &file_resid, vector<int> &indicator_resid,
+                  gsl_matrix *resid, size_t &n_resid);
 
 bool BimbamKin(const string file_geno, const set<string> ksnps,
                vector<int> &indicator_snp, const int k_mode,

src/lmm.cpp

@@ -49,8 +49,6 @@
 #include "lmm.h"
 #include "mathfunc.h"
 
-#define P_YY_MIN 0.00000001
-
 using namespace std;
 
 void LMM::CopyFromParam(PARAM &cPar) {
@@ -524,8 +522,7 @@ double LogL_f(double l, void *params) {
   index_yy = GetabIndex(n_cvt + 2, n_cvt + 2, n_cvt);
   double P_yy = gsl_matrix_safe_get(Pab, nc_total, index_yy);
 
-  if (P_yy >= 0.0 && (P_yy < P_YY_MIN)) P_yy = P_YY_MIN; // control potential round-off
-
+  if (P_yy == 0.0) P_yy = 0.00000001; // control potential round-off
   if (is_check_mode() || is_debug_mode()) {
     // cerr << "P_yy is" << P_yy << endl;
     assert(!is_nan(P_yy));
@@ -851,7 +848,7 @@ double LogRL_f(double l, void *params) {
   index_ww = GetabIndex(n_cvt + 2, n_cvt + 2, n_cvt);
   double P_yy = gsl_matrix_safe_get(Pab, nc_total, index_ww);
   // P_yy is positive and may get zeroed printf("P_yy=%f",P_yy);
-  if (P_yy >= 0.0 && (P_yy < P_YY_MIN)) P_yy = P_YY_MIN; // control potential round-off
+  if (P_yy == 0.0) P_yy = 0.00000001; // control potential round-off
 
   double c = 0.5 * df * (safe_log(df) - safe_log(2 * M_PI) - 1.0);
   f = c - 0.5 * logdet_h - 0.5 * logdet_hiw - 0.5 * df * safe_log(P_yy);
@@ -2027,14 +2024,13 @@ void CalcLambda(const char func_name, FUNC_PARAM &params, const double l_min,
     for (vector<double>::size_type i = 0; i < lambda_lh.size(); ++i) {
       lambda_l = lambda_lh[i].first;
       lambda_h = lambda_lh[i].second;
-      // printf("%f,%f\n",lambda_l,lambda_h);
-      auto handler = gsl_set_error_handler_off();
       gsl_root_fsolver_set((gsl_root_fsolver*)s_f, &F, lambda_l, lambda_h);
 
       int status = GSL_FAILURE;
       uint iter = 0;
       const auto max_iter = 100;
 
+      auto handler = gsl_set_error_handler_off();
       do {
         iter++;
         status = gsl_root_fsolver_iterate((gsl_root_fsolver*)s_f);