Merge pull request #55 from lokeshh/missing_values

v0dro · web-flow · commit ccfcc5797e17 · 2016-08-31T18:58:27.000+05:30
Remove deprecated missing value functions
diff --git a/benchmarks/correlation_matrix_methods/correlation_matrix.rb b/benchmarks/correlation_matrix_methods/correlation_matrix.rb
@@ -60,7 +60,6 @@ def prediction_optimized(vars,cases)
 
 rs[:c_v] = rs.collect {|row| row[:cases]*row[:vars]}
 
-rs.update
 rs.save("correlation_matrix.ds")
 Statsample::Excel.write(rs,"correlation_matrix.xls")
 
diff --git a/examples/correlation_matrix.rb b/examples/correlation_matrix.rb
@@ -7,18 +7,6 @@
 
 require 'statsample'
 Statsample::Analysis.store("Statsample::Bivariate.correlation_matrix") do
-  # It so happens that Daru::Vector and Daru::DataFrame must update metadata
-  # like positions of missing values every time they are created. 
-  #
-  # Since we dont have any missing values in the data that we are creating, 
-  # we set Daru.lazy_update = true so that missing data is not updated every
-  # time and things happen much faster.
-  #
-  # In case you do have missing data and lazy_update has been set to *true*, 
-  # you _SHOULD_ called `#update` on the concerned Vector or DataFrame object
-  # everytime an assingment or deletion cycle is complete.
-  Daru.lazy_update = true
-  
   # Create a Daru::DataFrame containing 4 vectors a, b, c and d.
   #
   # Notice that the `clone` option has been set to *false*. This tells Daru
@@ -36,10 +24,6 @@
   # Calculate correlation matrix by calling the `cor` shorthand.
   cm = cor(ds)
   summary(cm)
-
-  # Set lazy_update to *false* once our job is done so that this analysis does
-  # not accidentally affect code elsewhere.
-  Daru.lazy_update = false
 end
 
 if __FILE__==$0
diff --git a/examples/dataset.rb b/examples/dataset.rb
@@ -6,10 +6,6 @@
 require 'statsample'
 
 Statsample::Analysis.store(Daru::DataFrame) do
-  # We set lazy_update to *true* so that time is not wasted in updating
-  # metdata every time an assignment happens.
-  Daru.lazy_update = true
-
   samples = 1000
 
   # The 'new_with_size' function lets you specify the size of the 
@@ -26,9 +22,6 @@
   # order by default.
   ds = Daru::DataFrame.new({:a=>a,:b=>b}, order: [:b, :a])
   summary(ds)
-
-  # Reset lazy_update to *false* to prevent other code from breaking.
-  Daru.lazy_update = false
 end
 
 if __FILE__==$0
diff --git a/examples/dominance_analysis_bootstrap.rb b/examples/dominance_analysis_bootstrap.rb
@@ -3,10 +3,6 @@
 require 'statsample'
 
 Statsample::Analysis.store(Statsample::DominanceAnalysis::Bootstrap) do
-  # Remember to call *update* after an assignment/deletion cycle if lazy_update
-  # is *false*.
-  Daru.lazy_update = true
-
   sample=300
   a=rnorm(sample)
   b=rnorm(sample)
@@ -29,8 +25,6 @@
   dab2=dominance_analysis_bootstrap(ds2, :y1, :debug=>true)
   dab2.bootstrap(100,nil)
   summary(dab2)
-
-  Daru.lazy_update = false
 end
 
 if __FILE__==$0
diff --git a/examples/reliability.rb b/examples/reliability.rb
@@ -15,8 +15,6 @@
     ds["v#{i}".to_sym]= a + rnorm(samples,0,0.2)
   end
   
-  ds.update
-  
   rel=Statsample::Reliability::ScaleAnalysis.new(ds)
   summary rel
   
diff --git a/lib/statsample.rb b/lib/statsample.rb
@@ -206,15 +206,15 @@ def vector_cols_matrix(*vs)
     def only_valid(*vs)
       i = 1
       h = vs.inject({}) { |acc, v| acc["v#{i}".to_sym] = v; i += 1; acc }
-      df = Daru::DataFrame.new(h).dup_only_valid
+      df = Daru::DataFrame.new(h).reject_values(*Daru::MISSING_VALUES)
       df.map { |v| v }
     end
 
     # Cheap version of #only_valid.
     # If any vectors have missing_values, return only valid.
     # If not, return the vectors itself
     def only_valid_clone(*vs)
-      if vs.any?(&:has_missing_data?)
+      if vs.any? { |v| v.include_values?(*Daru::MISSING_VALUES) }
         only_valid(*vs)
       else
         vs
diff --git a/lib/statsample/anova/oneway.rb b/lib/statsample/anova/oneway.rb
@@ -164,7 +164,7 @@ def report_building(builder) # :nodoc:
           if summary_descriptives
             s.table(:name=>_("Descriptives"),:header=>%w{Name N Mean SD Min Max}.map {|v| _(v)}) do |t|
               @vectors.each do |v|
-                t.row [v.name, v.n_valid, "%0.4f" % v.mean, "%0.4f" %  v.sd, "%0.4f" % v.min, "%0.4f" % v.max]
+                t.row [v.name, v.reject_values(*Daru::MISSING_VALUES).size, "%0.4f" % v.mean, "%0.4f" %  v.sd, "%0.4f" % v.min, "%0.4f" % v.max]
               end
             end
           end
diff --git a/lib/statsample/bivariate.rb b/lib/statsample/bivariate.rb
@@ -159,7 +159,7 @@ def covariance_matrix_optimized(ds)
       
       def covariance_matrix(ds)
         vars,cases = ds.ncols, ds.nrows
-        if !ds.has_missing_data? and Statsample.has_gsl? and prediction_optimized(vars,cases) < prediction_pairwise(vars,cases)
+        if !ds.include_values?(*Daru::MISSING_VALUES) and Statsample.has_gsl? and prediction_optimized(vars,cases) < prediction_pairwise(vars,cases)
           cm=covariance_matrix_optimized(ds)
         else
           cm=covariance_matrix_pairwise(ds)
@@ -198,7 +198,7 @@ def covariance_matrix_pairwise(ds)
       # Order of rows and columns depends on Dataset#fields order
       def correlation_matrix(ds)
         vars, cases = ds.ncols, ds.nrows
-        if !ds.has_missing_data? and Statsample.has_gsl? and prediction_optimized(vars,cases) < prediction_pairwise(vars,cases)
+        if !ds.include_values?(*Daru::MISSING_VALUES) and Statsample.has_gsl? and prediction_optimized(vars,cases) < prediction_pairwise(vars,cases)
           cm=correlation_matrix_optimized(ds)
         else
           cm=correlation_matrix_pairwise(ds)
@@ -248,7 +248,7 @@ def n_valid_matrix(ds)
         m = vectors.collect do |row|
           vectors.collect do |col|
             if row==col
-              ds[row].only_valid.size
+              ds[row].reject_values(*Daru::MISSING_VALUES).size
             else
               rowa,rowb = Statsample.only_valid_clone(ds[row],ds[col])
               rowa.size
@@ -281,7 +281,7 @@ def spearman(v1,v2)
       # Calculate Point biserial correlation. Equal to Pearson correlation, with
       # one dichotomous value replaced by "0" and the other by "1"
       def point_biserial(dichotomous,continous)
-        ds = Daru::DataFrame.new({:d=>dichotomous,:c=>continous}).dup_only_valid
+        ds = Daru::DataFrame.new({:d=>dichotomous,:c=>continous}).reject_values(*Daru::MISSING_VALUES)
         raise(TypeError, "First vector should be dichotomous") if ds[:d].factors.size != 2
         raise(TypeError, "Second vector should be continous") if ds[:c].type != :numeric
         f0=ds[:d].factors.sort.to_a[0]
diff --git a/lib/statsample/converter/spss.rb b/lib/statsample/converter/spss.rb
@@ -7,7 +7,7 @@ class << self
       #   ds=Daru::DataFrame.from_excel("my_data.xls")
       #   puts Statsample::SPSS.tetrachoric_correlation_matrix(ds)
       def tetrachoric_correlation_matrix(ds)
-        dsv=ds.dup_only_valid
+        dsv=ds.reject_values(*Daru::MISSING_VALUES)
         # Delete all vectors doesn't have variation
         dsv.vectors.each { |f|
           if dsv[f].factors.size==1
diff --git a/lib/statsample/daru.rb b/lib/statsample/daru.rb
@@ -11,7 +11,7 @@ def histogram(bins=10)
         # ugly patch. The upper limit for a bin has the form
         # x < range
         #h=Statsample::Histogram.new(self, bins)
-        valid = only_valid
+        valid = reject_values(*Daru::MISSING_VALUES)
         min,max=Statsample::Util.nice(valid.min,valid.max)
         # fix last data
         if max == valid.max
@@ -72,7 +72,6 @@ def to_multiset_by_split_one_field(field)
       end
       #puts "Ingreso a los dataset"
       ms.datasets.each do |k,ds|
-        ds.update
         ds.rename self[field].index_of(k)
       end
 
@@ -102,7 +101,6 @@ def to_multiset_by_split_multiple_fields(*fields)
       each_row { |r| p1.call(r) }
 
       ms.datasets.each do |k,ds|
-        ds.update 
         ds.rename(
           fields.size.times.map do |i|
             f  = fields[i]
diff --git a/lib/statsample/factor/parallelanalysis.rb b/lib/statsample/factor/parallelanalysis.rb
@@ -142,8 +142,7 @@ def compute
                 raise "bootstrap_method doesn't recogniced"
               end
             end
-            ds_bootstrap.update
-            
+
             matrix=Statsample::Bivariate.send(matrix_method, ds_bootstrap)
             matrix=matrix.to_gsl if @use_gsl
             if smc
@@ -159,7 +158,6 @@ def compute
             redo
           end
         end
-        @ds_eigenvalues.update
       end
       dirty_memoize :number_of_factors, :ds_eigenvalues
       dirty_writer :iterations, :bootstrap_method, :percentil, :smc
diff --git a/lib/statsample/matrix.rb b/lib/statsample/matrix.rb
@@ -21,7 +21,6 @@ def to_dataframe
     row_size.times {|i|
       ds.add_row(self.row(i).to_a)
     }
-    ds.update
     ds.rename(self.name) if self.respond_to? :name
     ds
   end
@@ -95,7 +94,6 @@ def to_dataframe
       row_size.times {|i|
         ds.add_row(self.row(i).to_a)
       }
-      ds.update
       ds.rename(self.name) if self.respond_to? :name
       ds
     end
diff --git a/lib/statsample/regression.rb b/lib/statsample/regression.rb
@@ -56,7 +56,7 @@ def self.multiple(ds,y_var, opts=Hash.new)
           if Statsample.has_gsl? and false
             Statsample::Regression::Multiple::GslEngine.new(ds, y_var, opts)
           else
-            ds2=ds.dup_only_valid
+            ds2=ds.reject_values(*Daru::MISSING_VALUES)
             Statsample::Regression::Multiple::RubyEngine.new(ds2,y_var, opts)
           end
         end
diff --git a/lib/statsample/regression/multiple/alglibengine.rb b/lib/statsample/regression/multiple/alglibengine.rb
@@ -19,7 +19,7 @@ module Multiple
 class AlglibEngine < BaseEngine
   def initialize(ds,y_var, opts=Hash.new)
     super    
-    @ds       = ds.dup_only_valid
+    @ds       = ds.reject_values(*Daru::MISSING_VALUES)
     @ds_valid = @ds
     @dy       = @ds[@y_var]
     @ds_indep = ds.dup(ds.vectors.to_a - [y_var])
diff --git a/lib/statsample/regression/multiple/gslengine.rb b/lib/statsample/regression/multiple/gslengine.rb
@@ -19,7 +19,7 @@ module Multiple
         class GslEngine < BaseEngine
           def initialize(ds,y_var, opts=Hash.new)
             super
-            @ds          = ds.dup_only_valid
+            @ds          = ds.reject_values(*Daru::MISSING_VALUES)
             @ds_valid    = @ds
             @valid_cases = @ds_valid.nrows
             @dy          = @ds[@y_var]
diff --git a/lib/statsample/regression/multiple/rubyengine.rb b/lib/statsample/regression/multiple/rubyengine.rb
@@ -30,7 +30,7 @@ def initialize(ds,y_var, opts=Hash.new)
     super(matrix, y_var, opts)
     @ds = ds
     @dy = ds[@y_var]
-    @ds_valid = ds.dup_only_valid
+    @ds_valid = ds.reject_values(*Daru::MISSING_VALUES)
     @total_cases = @ds.nrows
     @valid_cases = @ds_valid.nrows
     @ds_indep    = ds.dup(ds.vectors.to_a - [y_var])
@@ -55,7 +55,6 @@ def fix_with_mean
       end
       i += 1
     end
-    @ds_indep.update
     set_dep_columns
   end
   def fix_with_regression
@@ -75,7 +74,6 @@ def fix_with_regression
       end
       i+=1
     end
-    @ds_indep.update
     set_dep_columns
   end
   # Standard error for constant
diff --git a/lib/statsample/reliability.rb b/lib/statsample/reliability.rb
@@ -4,7 +4,7 @@ class << self
       # Calculate Chonbach's alpha for a given dataset.
       # only uses tuples without missing data
       def cronbach_alpha(ods)
-        ds = ods.dup_only_valid
+        ds = ods.reject_values(*Daru::MISSING_VALUES)
         n_items = ds.ncols
         return nil if n_items <= 1
         s2_items = ds.to_hash.values.inject(0) { |ac,v| 
@@ -18,7 +18,7 @@ def cronbach_alpha(ods)
       # Only uses tuples without missing data
       # Return nil if one or more vectors has 0 variance
       def cronbach_alpha_standarized(ods)
-        ds = ods.dup_only_valid
+        ds = ods.reject_values(*Daru::MISSING_VALUES)
         return nil if ds.any? { |v| v.variance==0}
         
         ds = Daru::DataFrame.new(
diff --git a/lib/statsample/reliability/icc.rb b/lib/statsample/reliability/icc.rb
@@ -96,8 +96,7 @@ class ICC
       attr_accessor :alpha
       attr_accessor :name
       def initialize(ds, opts=Hash.new)
-        ds.update
-        @ds=ds.dup_only_valid
+        @ds=ds.reject_values(*Daru::MISSING_VALUES)
         @vectors=@ds.map { |e| e }
         @n=@ds.nrows
         @k=@ds.ncols
diff --git a/lib/statsample/reliability/multiscaleanalysis.rb b/lib/statsample/reliability/multiscaleanalysis.rb
@@ -128,7 +128,6 @@ def dataset_from_scales
           ds[code.to_sym] = scale.ds.vector_sum
         end
         
-        ds.update
         ds
       end
 
diff --git a/lib/statsample/reliability/scaleanalysis.rb b/lib/statsample/reliability/scaleanalysis.rb
@@ -21,7 +21,7 @@ def initialize(ds, opts=Hash.new)
         }
         
         @ods = ds
-        @ds  = ds.dup_only_valid(ds.vectors.to_a - @dumped)
+        @ds  = ds.reject_values(*Daru::MISSING_VALUES).dup(ds.vectors.to_a - @dumped)
         @ds.rename ds.name
         
         @k     = @ds.ncols
@@ -121,7 +121,6 @@ def item_difficulty_analysis
           dif_sort.each{ |variable,dif_value| row.push(case_row[variable]) }
           ds_new.add_row(row)
         end
-        ds_new.update
         ds_new
       end
       
diff --git a/lib/statsample/reliability/skillscaleanalysis.rb b/lib/statsample/reliability/skillscaleanalysis.rb
@@ -63,7 +63,7 @@ def corrected_dataset
             out = {}
             row.each_with_index do |v, k|
               if @key.has_key? k
-                if @ds[k].exists? v
+                if @ds[k].reject_values(*Daru::MISSING_VALUES).include_values? v
                   out[k]= @key[k] == v ? 1 : 0
                 else
                   out[k] = nil
diff --git a/lib/statsample/test/levene.rb b/lib/statsample/test/levene.rb
@@ -48,7 +48,7 @@ def report_building(builder) # :nodoc:
         builder.text "%s : F(%d, %d) = %0.4f , p = %0.4f" % [@name, @d1, @d2, f, probability]
       end
       def compute
-        n=@vectors.inject(0) { |ac,v| ac + v.n_valid}
+        n=@vectors.inject(0) { |ac,v| ac + v.reject_values(*Daru::MISSING_VALUES).size }
         
         zi=@vectors.collect do |vector|
           mean=vector.mean
@@ -57,7 +57,7 @@ def compute
         
         total_mean = Daru::Vector.new(
           zi.inject([]) do |ac,vector|
-            ac + vector.only_valid(:array)
+            ac + vector.reject_values(*Daru::MISSING_VALUES).to_a
           end
         ).mean
       
@@ -68,7 +68,7 @@ def compute
         
         sum_den = zi.inject(0) do |ac,vector|
           z_mean = vector.mean
-          ac + vector.only_valid(:array).inject(0) do |acp,zij|
+          ac + vector.reject_values(*Daru::MISSING_VALUES).to_a.inject(0) do |acp,zij|
             acp + (zij - z_mean)**2
           end
         end
diff --git a/lib/statsample/test/t.rb b/lib/statsample/test/t.rb
diff --git a/lib/statsample/test/umannwhitney.rb b/lib/statsample/test/umannwhitney.rb
diff --git a/statsample.gemspec b/statsample.gemspec
diff --git a/test/test_factor.rb b/test/test_factor.rb
diff --git a/test/test_factor_pa.rb b/test/test_factor_pa.rb
diff --git a/test/test_reliability.rb b/test/test_reliability.rb
diff --git a/test/test_statistics.rb b/test/test_statistics.rb
