diff --git a/reference.c b/reference.c
index 4cf8809..b49d4c7 100644
--- a/reference.c
+++ b/reference.c
@@ -1076,7 +1076,7 @@ REF_SetReference(int stratum,
   }
 
   /* Update fallback drifts */
-  if (fb_drifts) {
+  if (fb_drifts && are_we_synchronised) {
     update_fb_drifts(abs_freq_ppm, update_interval);
     schedule_fb_drift(&now);
   }