Merge branch 'next'

configure.ac

@@ -8,7 +8,7 @@ AC_PREREQ(2.59)
 # VERSION define is not used by the code.  It gets a version string
 # from 'git describe'; see src/ceph_ver.[ch]
 
-AC_INIT([ceph], [0.72-rc1], [[email protected]])
+AC_INIT([ceph], [0.72], [[email protected]])
 
 # Create release string.  Used with VERSION for RPMs.
 RPM_RELEASE=0

debian/changelog

@@ -1,3 +1,9 @@
+ceph (0.72-1) stable; urgency=low
+
+  * New upstream release
+
+ -- Gary Lowell <[email protected]>  Thu, 07 Nov 2013 20:25:18 +0000
+
 ceph (0.72-rc1-1) stable; urgency=low
 
   * New upstream release 

doc/release-notes.rst

@@ -407,6 +407,12 @@ v0.69
 Upgrading
 ~~~~~~~~~
 
+* The sysvinit /etc/init.d/ceph script will, by default, update the
+  CRUSH location of an OSD when it starts.  Previously, if the
+  monitors were not available, this command would hang indefinitely.
+  Now, that step will time out after 10 seconds and the ceph-osd daemon
+  will not be started.
+
 * Users of the librados C++ API should replace users of get_version()
   with get_version64() as the old method only returns a 32-bit value
   for a 64-bit field.  The existing 32-bit get_version() method is now

qa/run_xfstests.sh

@@ -48,7 +48,7 @@ XFS_MKFS_OPTIONS="-l su=32k"
 
 # Override the default test list with a list of tests known to pass
 # until we can work through getting them all passing reliably.
-TESTS="1-9 11-15 17 19-21 26-29 31-34 41 46-48 50-54 56 61 63-67 69-70 74-76"
+TESTS="1-7 9 11-15 17 19-21 26-29 31-34 41 46-48 50-54 56 61 63-67 69-70 74-76"
 TESTS="${TESTS} 78 79 84-89 91-92 100 103 105 108 110 116-121 124 126"
 TESTS="${TESTS} 129-135 137-141 164-167 182 184 187-190 192 194"
 TESTS="${TESTS} 196 199 201 203 214-216 220-227 234 236-238 241 243-249"
@@ -59,6 +59,9 @@ TESTS="${TESTS} 253 257-259 261 262 269 273 275 277 278 280 285 286"
 ######
 # Some explanation of why tests have been excluded above:
 #
+# Test 008 was pulled because it contained a race condition leading to
+#          spurious failures.
+#
 # Test 049 was pulled because it caused a kernel fault.
 #	http://tracker.newdream.net/issues/2260
 # Test 232 was pulled because it caused an XFS error

qa/run_xfstests_qemu.sh

@@ -7,4 +7,4 @@ chmod +x run_xfstests.sh
 # tests excluded fail in the current testing vm regardless of whether
 # rbd is used
 
-./run_xfstests.sh -c 1 -f xfs -t /dev/vdb -s /dev/vdc 1-17 19-26 28-49 51-61 63 66-67 69-79 83 85-105 108-110 112-135 137-170 174-191 193-204 206-217 220-227 230-231 233 235-241 243-249 251-262 264-278 281-286 288-289
+./run_xfstests.sh -c 1 -f xfs -t /dev/vdb -s /dev/vdc 1-7 9-17 19-26 28-49 51-61 63 66-67 69-79 83 85-105 108-110 112-135 137-170 174-191 193-204 206-217 220-227 230-231 233 235-241 243-249 251-262 264-278 281-286 288-289

src/common/hobject.h

@@ -86,6 +86,11 @@ struct hobject_t {
     return ret;
   }
 
+  /// @return true if object is snapdir
+  bool is_snapdir() const {
+    return snap == CEPH_SNAPDIR;
+  }
+
   /// @return snapdir version of this hobject_t
   hobject_t get_snapdir() const {
     hobject_t ret(*this);

src/common/sharedptr_registry.hpp

@@ -149,6 +149,11 @@ class SharedPtrRegistry {
     return retval;
   }
 
+  unsigned size() {
+    Mutex::Locker l(lock);
+    return contents.size();
+  }
+
   void remove(const K &key) {
     Mutex::Locker l(lock);
     contents.erase(key);

src/mon/OSDMonitor.cc

@@ -2796,10 +2796,15 @@ int OSDMonitor::prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
     }
     if (n <= (int)p.get_pg_num()) {
       ss << "specified pg_num " << n << " <= current " << p.get_pg_num();
-    } else if (!mon->pgmon()->pg_map.creating_pgs.empty()) {
-      ss << "currently creating pgs, wait";
-      return -EAGAIN;
     } else {
+      for(set<pg_t>::iterator i = mon->pgmon()->pg_map.creating_pgs.begin();
+	  i != mon->pgmon()->pg_map.creating_pgs.end();
+	  ++i) {
+	if (i->m_pool == static_cast<uint64_t>(pool)) {
+	  ss << "currently creating pgs, wait";
+	  return -EAGAIN;
+	}
+      }
       p.set_pg_num(n);
       ss << "set pool " << pool << " pg_num to " << n;
     }
@@ -2812,10 +2817,15 @@ int OSDMonitor::prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
       ss << "specified pgp_num must > 0, but you set to " << n;
     } else if (n > (int)p.get_pg_num()) {
       ss << "specified pgp_num " << n << " > pg_num " << p.get_pg_num();
-    } else if (!mon->pgmon()->pg_map.creating_pgs.empty()) {
-      ss << "still creating pgs, wait";
-      return -EAGAIN;
     } else {
+      for(set<pg_t>::iterator i = mon->pgmon()->pg_map.creating_pgs.begin();
+	  i != mon->pgmon()->pg_map.creating_pgs.end();
+	  ++i) {
+	if (i->m_pool == static_cast<uint64_t>(pool)) {
+	  ss << "currently creating pgs, wait";
+	  return -EAGAIN;
+	}
+      }
       p.set_pgp_num(n);
       ss << "set pool " << pool << " pgp_num to " << n;
     }

src/mon/PGMap.cc

@@ -787,7 +787,7 @@ void PGMap::print_osd_perf_stats(std::ostream *ss) const
 }
 
 void PGMap::recovery_summary(Formatter *f, ostream *out,
-                             pool_stat_t delta_sum) const
+                             const pool_stat_t& delta_sum) const
 {
   bool first = true;
   if (delta_sum.stats.sum.num_objects_degraded) {
@@ -825,7 +825,7 @@ void PGMap::recovery_summary(Formatter *f, ostream *out,
 }
 
 void PGMap::recovery_rate_summary(Formatter *f, ostream *out,
-                                  pool_stat_t delta_sum,
+                                  const pool_stat_t& delta_sum,
                                   utime_t delta_stamp) const
 {
   // make non-negative; we can get negative values if osds send
@@ -886,7 +886,7 @@ void PGMap::pool_recovery_summary(Formatter *f, ostream *out,
 }
 
 void PGMap::client_io_rate_summary(Formatter *f, ostream *out,
-                                   pool_stat_t delta_sum,
+                                   const pool_stat_t& delta_sum,
                                    utime_t delta_stamp) const
 {
   pool_stat_t pos_delta = delta_sum;

src/mon/PGMap.h

@@ -244,12 +244,12 @@ class PGMap {
   void print_osd_perf_stats(std::ostream *ss) const;
 
   void recovery_summary(Formatter *f, ostream *out,
-                        pool_stat_t delta_sum) const;
+                        const pool_stat_t& delta_sum) const;
   void overall_recovery_summary(Formatter *f, ostream *out) const;
   void pool_recovery_summary(Formatter *f, ostream *out,
                              uint64_t poolid) const;
   void recovery_rate_summary(Formatter *f, ostream *out,
-                             pool_stat_t delta_sum,
+                             const pool_stat_t& delta_sum,
                              utime_t delta_stamp) const;
   void overall_recovery_rate_summary(Formatter *f, ostream *out) const;
   void pool_recovery_rate_summary(Formatter *f, ostream *out,
@@ -259,7 +259,7 @@ class PGMap {
    * given @p delta_sum pool over a given @p delta_stamp period of time.
    */
   void client_io_rate_summary(Formatter *f, ostream *out,
-                              pool_stat_t delta_sum,
+                              const pool_stat_t& delta_sum,
                               utime_t delta_stamp) const;
   /**
    * Obtain a formatted/plain output for the overall client I/O, which is

src/os/FileStore.cc

@@ -4273,6 +4273,14 @@ int FileStore::_collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
   int r = 0;
   int dstcmp, srccmp;
 
+  if (replaying) {
+    /* If the destination collection doesn't exist during replay,
+     * we need to delete the src object and continue on
+     */
+    if (!collection_exists(c))
+      goto out_rm_src;
+  }
+
   dstcmp = _check_replay_guard(c, o, spos);
   if (dstcmp < 0)
     goto out_rm_src;

src/os/chain_xattr.cc

@@ -388,6 +388,10 @@ int chain_listxattr(const char *fn, char *names, size_t len) {
 
 int chain_flistxattr(int fd, char *names, size_t len) {
   int r;
+  char *p;
+  const char * end;
+  char *dest;
+  char *dest_end;
 
   if (!len)
     return sys_flistxattr(fd, names, len) * 2;
@@ -403,12 +407,12 @@ int chain_flistxattr(int fd, char *names, size_t len) {
 
   r = sys_flistxattr(fd, full_buf, total_len);
   if (r < 0)
-    return r;
+    goto done;
 
-  char *p = full_buf;
-  const char *end = full_buf + r;
-  char *dest = names;
-  char *dest_end = names + len;
+  p = full_buf;
+  end = full_buf + r;
+  dest = names;
+  dest_end = names + len;
 
   while (p < end) {
     char name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];

src/osd/ErasureCodePluginJerasure/galois.c

@@ -693,7 +693,7 @@ void galois_w32_region_multiply(char *region,      /* Region to multiply */
   nbytes /= sizeof(int);
 
   if (galois_split_w8[0]== NULL) {
-    if (galois_create_split_w8_tables(8) < 0) {
+    if (galois_create_split_w8_tables() < 0) {
       fprintf(stderr, "galois_32_region_multiply -- couldn't make split multiplication tables\n");
       exit(1);
     }

src/osd/OSD.cc

@@ -2201,9 +2201,10 @@ void OSD::handle_pg_peering_evt(
     int role = osdmap->calc_pg_role(whoami, acting, acting.size());
 
     pg_history_t history = info.history;
-    project_pg_history(info.pgid, history, epoch, up, acting);
+    bool valid_history = project_pg_history(
+      info.pgid, history, epoch, up, acting);
 
-    if (epoch < history.same_interval_since) {
+    if (!valid_history || epoch < history.same_interval_since) {
       dout(10) << "get_or_create_pg " << info.pgid << " acting changed in "
 	       << history.same_interval_since << " (msg from " << epoch << ")" << dendl;
       return;
@@ -2388,7 +2389,7 @@ void OSD::calc_priors_during(pg_t pgid, epoch_t start, epoch_t end, set<int>& ps
  * Fill in the passed history so you know same_interval_since, same_up_since,
  * and same_primary_since.
  */
-void OSD::project_pg_history(pg_t pgid, pg_history_t& h, epoch_t from,
+bool OSD::project_pg_history(pg_t pgid, pg_history_t& h, epoch_t from,
 			     const vector<int>& currentup,
 			     const vector<int>& currentacting)
 {
@@ -2402,7 +2403,11 @@ void OSD::project_pg_history(pg_t pgid, pg_history_t& h, epoch_t from,
        e > from;
        e--) {
     // verify during intermediate epoch (e-1)
-    OSDMapRef oldmap = get_map(e-1);
+    OSDMapRef oldmap = service.try_get_map(e-1);
+    if (!oldmap) {
+      dout(15) << __func__ << ": found map gap, returning false" << dendl;
+      return false;
+    }
     assert(oldmap->have_pg_pool(pgid.pool()));
 
     vector<int> up, acting;
@@ -2452,6 +2457,7 @@ void OSD::project_pg_history(pg_t pgid, pg_history_t& h, epoch_t from,
   }
 
   dout(15) << "project_pg_history end " << h << dendl;
+  return true;
 }
 
 // -------------------------------------
@@ -5441,22 +5447,6 @@ void OSD::advance_map(ObjectStore::Transaction& t, C_Contexts *tfin)
       waiting_for_pg.erase(p++);
     }
   }
-  map<pg_t, list<PG::CephPeeringEvtRef> >::iterator q =
-    peering_wait_for_split.begin();
-  while (q != peering_wait_for_split.end()) {
-    pg_t pgid = q->first;
-
-    // am i still primary?
-    vector<int> acting;
-    int nrep = osdmap->pg_to_acting_osds(pgid, acting);
-    int role = osdmap->calc_pg_role(whoami, acting, nrep);
-    if (role >= 0) {
-      ++q;  // still me
-    } else {
-      dout(10) << " discarding waiting ops for " << pgid << dendl;
-      peering_wait_for_split.erase(q++);
-    }
-  }
 }
 
 void OSD::consume_map()
@@ -5935,7 +5925,12 @@ void OSD::handle_pg_create(OpRequestRef op)
     utime_t now = ceph_clock_now(NULL);
     history.last_scrub_stamp = now;
     history.last_deep_scrub_stamp = now;
-    project_pg_history(pgid, history, created, up, acting);
+    bool valid_history =
+      project_pg_history(pgid, history, created, up, acting);
+    /* the pg creation message must have come from a mon and therefore
+     * cannot be on the other side of a map gap
+     */
+    assert(valid_history);
 
     // register.
     creating_pgs[pgid].history = history;
@@ -6547,9 +6542,11 @@ void OSD::handle_pg_query(OpRequestRef op)
 
     // same primary?
     pg_history_t history = it->second.history;
-    project_pg_history(pgid, history, it->second.epoch_sent, up, acting);
+    bool valid_history =
+      project_pg_history(pgid, history, it->second.epoch_sent, up, acting);
 
-    if (it->second.epoch_sent < history.same_interval_since) {
+    if (!valid_history ||
+        it->second.epoch_sent < history.same_interval_since) {
       dout(10) << " pg " << pgid << " dne, and pg has changed in "
 	       << history.same_interval_since
 	       << " (msg from " << it->second.epoch_sent << ")" << dendl;
@@ -6613,9 +6610,11 @@ void OSD::handle_pg_remove(OpRequestRef op)
     pg_history_t history = pg->info.history;
     vector<int> up, acting;
     osdmap->pg_to_up_acting_osds(pgid, up, acting);
-    project_pg_history(pg->info.pgid, history, pg->get_osdmap()->get_epoch(),
-		       up, acting);
-    if (history.same_interval_since <= m->get_epoch()) {
+    bool valid_history =
+      project_pg_history(pg->info.pgid, history, pg->get_osdmap()->get_epoch(),
+	up, acting);
+    if (valid_history &&
+        history.same_interval_since <= m->get_epoch()) {
       assert(pg->get_primary() == m->get_source().num());
       PGRef _pg(pg);
       _remove_pg(pg);

src/osd/OSD.h

@@ -1191,8 +1191,12 @@ class OSD : public Dispatcher,
   void build_past_intervals_parallel();
 
   void calc_priors_during(pg_t pgid, epoch_t start, epoch_t end, set<int>& pset);
-  void project_pg_history(pg_t pgid, pg_history_t& h, epoch_t from,
-			  const vector<int>& lastup, const vector<int>& lastacting);
+
+  /// project pg history from from to now
+  bool project_pg_history(
+    pg_t pgid, pg_history_t& h, epoch_t from,
+    const vector<int>& lastup, const vector<int>& lastacting
+    ); ///< @return false if there was a map gap between from and now
 
   void wake_pg_waiters(pg_t pgid) {
     if (waiting_for_pg.count(pgid)) {

src/osd/PG.cc

@@ -2399,13 +2399,7 @@ void PG::log_weirdness()
 		      << " != info.last_update " << info.last_update
 		      << "\n";
 
-  if (pg_log.get_log().empty()) {
-    // shoudl it be?
-    if (pg_log.get_head() != pg_log.get_tail())
-      osd->clog.error() << info.pgid
-			<< " log bound mismatch, empty but (" << pg_log.get_tail() << ","
-			<< pg_log.get_head() << "]\n";
-  } else {
+  if (!pg_log.get_log().empty()) {
     // sloppy check
     if ((pg_log.get_log().log.begin()->version <= pg_log.get_tail()))
       osd->clog.error() << info.pgid
@@ -4679,19 +4673,11 @@ ostream& operator<<(ostream& out, const PG& pg)
       pg.pg_log.get_head() != pg.info.last_update)
     out << " (info mismatch, " << pg.pg_log.get_log() << ")";
 
-  if (pg.pg_log.get_log().empty()) {
-    // shoudl it be?
-    if (pg.pg_log.get_head().version - pg.pg_log.get_tail().version != 0) {
-      out << " (log bound mismatch, empty)";
-    }
-  } else {
-    if ((pg.pg_log.get_log().log.begin()->version <= pg.pg_log.get_tail()) || // sloppy check
-        (pg.pg_log.get_log().log.rbegin()->version != pg.pg_log.get_head() &&
-	 !(pg.pg_log.get_head() == pg.pg_log.get_tail()))) {
+  if (!pg.pg_log.get_log().empty()) {
+    if ((pg.pg_log.get_log().log.begin()->version <= pg.pg_log.get_tail())) {
       out << " (log bound mismatch, actual=["
 	  << pg.pg_log.get_log().log.begin()->version << ","
 	  << pg.pg_log.get_log().log.rbegin()->version << "]";
-      //out << "len=" << pg.log.log.size();
       out << ")";
     }
   }

src/osd/PGLog.cc

@@ -360,7 +360,7 @@ void PGLog::rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead
     }
     --p;
     mark_dirty_from(p->version);
-    if (p->version == newhead) {
+    if (p->version <= newhead) {
       ++p;
       divergent.splice(divergent.begin(), log.log, p, log.log.end());
       break;